@melra/runtime-core 0.3.0-alpha.3

package/dist/workflow-controller.js

@@ -0,0 +1,711 @@
+// Copyright 2026 XAGI Labs Private Limited
+// SPDX-License-Identifier: Apache-2.0
+import { randomUUID } from "node:crypto";
+import { WorkflowDefinitionSchema, WorkflowEventSchema, WorkflowRunSchema, WorkflowSnapshotSchema, } from "@melra/protocol";
+import { canonicalJson, redactStructuredValue, sha256, } from "@melra/receipt-schema";
+import { applyWorkflowEvent, rebuildWorkflow, } from "./workflow-events.js";
+import { readyNodeIds, validateWorkflow } from "./workflow-graph.js";
+export class WorkflowController {
+    store;
+    tasks;
+    payloadCipher;
+    // ponytail: process-local serialization; add SQLite leases before supporting
+    // multiple MELRA server processes against one data directory.
+    advances = new Map();
+    constructor(store, tasks, payloadCipher) {
+        this.store = store;
+        this.tasks = tasks;
+        this.payloadCipher = payloadCipher;
+    }
+    plan(definition) {
+        const parsed = WorkflowDefinitionSchema.parse(definition);
+        const graph = validateWorkflow(parsed);
+        for (const node of graph.nodes.values()) {
+            for (const request of requestsFor(node)) {
+                if (this.tasks.preflight(request).outcome === "deny") {
+                    throw new Error(`workflow_policy_blocked:${node.id}`);
+                }
+            }
+        }
+        const createdAt = new Date().toISOString();
+        const id = randomUUID();
+        const traceId = randomUUID();
+        const draft = WorkflowRunSchema.parse({
+            schemaVersion: "1.0.0",
+            id,
+            definitionId: parsed.id,
+            definitionVersion: parsed.version,
+            status: "draft",
+            stateVersion: 1,
+            nodes: Object.fromEntries(parsed.nodes.map((node) => [node.id, { status: "pending" }])),
+            traceId,
+            createdAt,
+            updatedAt: createdAt,
+        });
+        const created = makeEvent(draft, 1, "workflow.created", { run: draft });
+        const plannedEvent = makeEvent(draft, 2, "workflow.status_changed", {
+            from: "draft",
+            to: "planned",
+        });
+        const planned = applyWorkflowEvent(draft, plannedEvent);
+        const redacted = WorkflowDefinitionSchema.parse(redactStructuredValue(parsed).value);
+        this.store.createWorkflow(redacted, this.payloadCipher.seal(parsed, `workflow:${parsed.id}:${parsed.version}:definition`), planned, [created, plannedEvent]);
+        return planned;
+    }
+    status(workflowId) {
+        const run = this.store.getWorkflowRun(workflowId);
+        if (run === undefined)
+            throw new Error("workflow_not_found");
+        return run;
+    }
+    events(workflowId, afterSequence = 0) {
+        this.status(workflowId);
+        return this.store.listWorkflowEvents(workflowId, afterSequence);
+    }
+    async advance(workflowId, approvals = []) {
+        const previous = this.advances.get(workflowId);
+        const ready = previous === undefined
+            ? Promise.resolve()
+            : previous.then(() => undefined, () => undefined);
+        const result = ready.then(() => this.advanceOnce(workflowId, approvals));
+        this.advances.set(workflowId, result);
+        try {
+            return await result;
+        }
+        finally {
+            if (this.advances.get(workflowId) === result) {
+                this.advances.delete(workflowId);
+            }
+        }
+    }
+    async advanceOnce(workflowId, approvals) {
+        const current = this.status(workflowId);
+        if (["verified_complete", "cancelled"].includes(current.status)) {
+            return { run: current, tasks: [], events: [] };
+        }
+        const definition = this.loadDefinition(current);
+        const graph = validateWorkflow(definition);
+        const rollback = current.status === "failed"
+            ? definition.nodes
+                .filter((node) => node.type === "compensation" &&
+                ["pending", "awaiting_approval"].includes(current.nodes[node.id]?.status ?? "") &&
+                current.nodes[node.forNodeId]?.status ===
+                    "verified_complete")
+                .map((node) => node.id)
+                .reverse()
+            : [];
+        if (current.status === "failed" && rollback.length === 0) {
+            return { run: current, tasks: [], events: [] };
+        }
+        const awaiting = [...graph.nodes.values()]
+            .filter((node) => node.type !== "compensation" &&
+            current.nodes[node.id]?.status === "awaiting_approval")
+            .map((node) => node.id)
+            .sort();
+        const nodeIds = rollback.length > 0
+            ? rollback
+            : awaiting.length > 0
+                ? awaiting
+                : readyNodeIds(graph, current.nodes);
+        if (nodeIds.length === 0) {
+            return { run: current, tasks: [], events: [] };
+        }
+        const results = await Promise.all(nodeIds.map(async (nodeId) => {
+            const node = graph.nodes.get(nodeId);
+            return {
+                nodeId,
+                result: await this.advanceNode(node, definition, current.nodes[nodeId], current, approvals),
+            };
+        }));
+        if (results.some(({ result }) => result.state.status === "failed")) {
+            for (const node of [...definition.nodes].reverse()) {
+                if (node.type !== "compensation" ||
+                    current.nodes[node.id]?.status !== "pending") {
+                    continue;
+                }
+                const targetState = results.find(({ nodeId }) => nodeId === node.forNodeId)?.result
+                    .state ?? current.nodes[node.forNodeId];
+                if (targetState?.status !== "verified_complete") {
+                    results.push({
+                        nodeId: node.id,
+                        result: {
+                            state: { status: "skipped", taskIds: [] },
+                            tasks: [],
+                        },
+                    });
+                    continue;
+                }
+                const compensation = await this.runTask(node.request, current.nodes[node.id]?.taskIds[0], approvals, true, {
+                    workflowId: current.id,
+                    nodeId: node.id,
+                    iteration: 0,
+                    branch: "compensation",
+                });
+                results.push({
+                    nodeId: node.id,
+                    result: {
+                        ...compensation,
+                        state: compensation.state.status === "verified_complete"
+                            ? { ...compensation.state, status: "compensated" }
+                            : compensation.state,
+                    },
+                });
+            }
+        }
+        let run = this.transition(current, (draft, emit) => {
+            for (const { nodeId, result } of results) {
+                const from = draft.nodes[nodeId].status;
+                draft.nodes[nodeId] = result.state;
+                emit("workflow.node_changed", {
+                    nodeId,
+                    from,
+                    state: result.state,
+                });
+            }
+            let status = this.deriveStatus(draft, definition);
+            if (status === "verified_complete") {
+                for (const node of definition.nodes) {
+                    const state = draft.nodes[node.id];
+                    if (node.type !== "compensation" || state.status !== "pending") {
+                        continue;
+                    }
+                    state.status = "skipped";
+                    emit("workflow.node_changed", {
+                        nodeId: node.id,
+                        from: "pending",
+                        state,
+                    });
+                }
+                status = this.deriveStatus(draft, definition);
+            }
+            if (status !== draft.status) {
+                const from = draft.status;
+                draft.status = status;
+                emit("workflow.status_changed", { from, to: status });
+            }
+        });
+        if (results.some(({ result }) => result.checkpoint === true)) {
+            run = this.transition(run, (draft, emit) => {
+                emit("workflow.checkpoint_saved", {
+                    sequence: draft.stateVersion + 1,
+                });
+            });
+            this.store.saveWorkflowSnapshot(WorkflowSnapshotSchema.parse({
+                schemaVersion: "1.0.0",
+                workflowId: run.id,
+                sequence: run.stateVersion,
+                run,
+                createdAt: run.updatedAt,
+            }));
+        }
+        return {
+            run,
+            tasks: results.flatMap(({ result }) => result.tasks),
+            events: this.events(workflowId, current.stateVersion),
+        };
+    }
+    async recoverInterrupted() {
+        await this.tasks.recoverInterrupted();
+        const recovered = [];
+        const allTasks = this.store.listTasks(100_000);
+        for (const stored of this.store.listWorkflowRuns([
+            "planned",
+            "running",
+            "awaiting_approval",
+            "recovery_required",
+        ])) {
+            const replayed = this.replay(stored);
+            if (replayed.stateVersion !== stored.stateVersion) {
+                throw new Error("workflow_event_history_invalid");
+            }
+            const definition = this.loadDefinition(stored);
+            let run = this.transition(stored, (draft, emit) => {
+                for (const node of definition.nodes) {
+                    const state = recoveredNodeState(node, expectedTaskKeys(stored.id, node, definition)
+                        .map((key) => preferredTask(allTasks, key))
+                        .filter((task) => task !== undefined), replayed.nodes[node.id]);
+                    if (canonicalJson(draft.nodes[node.id]) === canonicalJson(state)) {
+                        continue;
+                    }
+                    const from = draft.nodes[node.id].status;
+                    draft.nodes[node.id] = state;
+                    emit("workflow.node_changed", {
+                        nodeId: node.id,
+                        from,
+                        state,
+                    });
+                }
+                let status = this.deriveStatus(draft, definition);
+                if (status === "verified_complete") {
+                    for (const node of definition.nodes) {
+                        const state = draft.nodes[node.id];
+                        if (node.type !== "compensation" || state.status !== "pending") {
+                            continue;
+                        }
+                        state.status = "skipped";
+                        emit("workflow.node_changed", {
+                            nodeId: node.id,
+                            from: "pending",
+                            state,
+                        });
+                    }
+                    status = this.deriveStatus(draft, definition);
+                }
+                const from = draft.status;
+                draft.status = status;
+                if (status === "recovery_required") {
+                    draft.error = "workflow_contains_unreconciled_mutation";
+                }
+                else {
+                    delete draft.error;
+                }
+                emit(status === "recovery_required"
+                    ? "workflow.recovery_required"
+                    : "workflow.recovered", {
+                    from,
+                    to: status,
+                    ...(draft.error === undefined ? {} : { error: draft.error }),
+                });
+            });
+            recovered.push(run);
+        }
+        return recovered;
+    }
+    cancel(workflowId) {
+        const current = this.status(workflowId);
+        if (current.status === "verified_complete" ||
+            current.status === "cancelled") {
+            return current;
+        }
+        return this.transition(current, (draft, emit) => {
+            for (const [nodeId, node] of Object.entries(draft.nodes)) {
+                if (!["pending", "ready", "awaiting_approval"].includes(node.status)) {
+                    continue;
+                }
+                for (const taskId of node.taskIds)
+                    this.tasks.cancel(taskId);
+                const from = node.status;
+                node.status = "cancelled";
+                emit("workflow.node_changed", {
+                    nodeId,
+                    from,
+                    state: node,
+                });
+            }
+            const from = draft.status;
+            draft.status = "cancelled";
+            delete draft.error;
+            emit("workflow.cancelled", { from, to: "cancelled" });
+        });
+    }
+    transition(current, mutate) {
+        const draft = structuredClone(current);
+        const events = [];
+        const emit = (type, data) => {
+            events.push(makeEvent(current, current.stateVersion + events.length + 1, type, data));
+        };
+        mutate(draft, emit);
+        if (events.length === 0)
+            return current;
+        draft.stateVersion = current.stateVersion + events.length;
+        draft.updatedAt = events.at(-1).occurredAt;
+        const expected = WorkflowRunSchema.parse(draft);
+        let rebuilt = current;
+        for (const event of events)
+            rebuilt = applyWorkflowEvent(rebuilt, event);
+        if (canonicalJson(rebuilt) !== canonicalJson(expected)) {
+            throw new Error("workflow_projection_event_mismatch");
+        }
+        this.store.transitionWorkflow(current.id, current.stateVersion, rebuilt, events);
+        return rebuilt;
+    }
+    loadDefinition(run) {
+        const payload = this.store.getWorkflowPayload(run.definitionId, run.definitionVersion);
+        if (payload === undefined)
+            throw new Error("workflow_payload_not_found");
+        return WorkflowDefinitionSchema.parse(this.payloadCipher.open(payload, `workflow:${run.definitionId}:${run.definitionVersion}:definition`));
+    }
+    replay(run) {
+        try {
+            const snapshot = this.store.getLatestWorkflowSnapshot(run.id);
+            return rebuildWorkflow(snapshot, this.store.listWorkflowEvents(run.id, snapshot?.sequence ?? 0));
+        }
+        catch {
+            try {
+                return rebuildWorkflow(undefined, this.store.listWorkflowEvents(run.id));
+            }
+            catch {
+                throw new Error("workflow_event_history_invalid");
+            }
+        }
+    }
+    async advanceNode(node, definition, state, run, approvals) {
+        switch (node.type) {
+            case "operation": {
+                const approvalNode = definition.nodes.find((candidate) => candidate.type === "approval" &&
+                    candidate.forNodeId === node.id);
+                const approvedTaskId = approvalNode === undefined
+                    ? undefined
+                    : run.nodes[approvalNode.id]?.taskIds[0];
+                return await this.runTask(node.request, approvedTaskId ?? state.taskIds[0], approvals, true, {
+                    workflowId: run.id,
+                    nodeId: node.id,
+                    iteration: 0,
+                    branch: "operation",
+                });
+            }
+            case "approval": {
+                const target = definition.nodes.find((candidate) => candidate.id === node.forNodeId);
+                if (target === undefined || target.type !== "operation") {
+                    throw new Error(`workflow_approval_target_missing:${node.id}`);
+                }
+                return await this.runTask(target.request, state.taskIds[0], approvals, false, {
+                    workflowId: run.id,
+                    nodeId: target.id,
+                    iteration: 0,
+                    branch: "operation",
+                });
+            }
+            case "condition": {
+                const sourceTaskId = run.nodes[node.sourceNodeId]?.taskIds[0];
+                if (sourceTaskId === undefined) {
+                    throw new Error(`workflow_condition_source_unavailable:${node.id}`);
+                }
+                const condition = await this.tasks.verifyPersisted(sourceTaskId, [
+                    node.predicate,
+                ]);
+                return await this.runSequential(condition.verified ? node.whenTrue : node.whenFalse, state, approvals, {
+                    workflowId: run.id,
+                    nodeId: node.id,
+                    iteration: 0,
+                    branch: condition.verified ? "true" : "false",
+                });
+            }
+            case "parallel":
+                return await this.runParallel(node.branches, state, approvals, run.id, node.id);
+            case "bounded_loop":
+                return await this.runLoop(node, state, approvals, run.id);
+            case "compensation": {
+                const result = await this.runTask(node.request, state.taskIds[0], approvals, true, {
+                    workflowId: run.id,
+                    nodeId: node.id,
+                    iteration: 0,
+                    branch: "compensation",
+                });
+                return {
+                    ...result,
+                    state: result.state.status === "verified_complete"
+                        ? { ...result.state, status: "compensated" }
+                        : result.state,
+                };
+            }
+            case "checkpoint":
+                return {
+                    state: { status: "verified_complete", taskIds: [] },
+                    tasks: [],
+                    checkpoint: true,
+                };
+            default:
+                throw new Error("workflow_node_not_executable");
+        }
+    }
+    async runTask(request, existingTaskId, approvals, executePlanned = true, identity) {
+        let task = existingTaskId === undefined
+            ? this.tasks.plan(request, {
+                idempotencyKey: taskIdempotencyKey(identity, request),
+                attempt: 1,
+            })
+            : this.tasks.status(existingTaskId);
+        if (task.status === "awaiting_approval") {
+            const approval = approvals.find((item) => item.approvalId === task.approval?.approvalId);
+            if (approval === undefined) {
+                return {
+                    state: {
+                        status: "awaiting_approval",
+                        taskIds: [task.id],
+                        ...(task.approval === undefined
+                            ? {}
+                            : { approval: task.approval }),
+                    },
+                    tasks: [task],
+                };
+            }
+            task = (await this.tasks.execute(task.id, approval)).task;
+        }
+        else if (task.status === "planned" && executePlanned) {
+            task = (await this.tasks.execute(task.id)).task;
+        }
+        return {
+            state: nodeStateForTask(task, !executePlanned),
+            tasks: [task],
+        };
+    }
+    async runSequential(requests, state, approvals, identity) {
+        const taskIds = [...state.taskIds];
+        const tasks = [];
+        for (const [index, request] of requests.entries()) {
+            const result = await this.runTask(request, taskIds[index], approvals, true, { ...identity, branch: `${identity.branch}:${index}` });
+            tasks.push(...result.tasks);
+            taskIds[index] = result.state.taskIds[0];
+            if (result.state.status !== "verified_complete") {
+                return {
+                    state: { ...result.state, taskIds },
+                    tasks,
+                };
+            }
+        }
+        return {
+            state: { status: "verified_complete", taskIds },
+            tasks,
+        };
+    }
+    async runParallel(branches, state, approvals, workflowId, nodeId) {
+        const requests = branches.flatMap((branch, branchIndex) => branch.map((request, requestIndex) => ({
+            request,
+            branch: `${branchIndex}:${requestIndex}`,
+        })));
+        const results = await Promise.all(requests.map((item, index) => this.runTask(item.request, state.taskIds[index], approvals, true, {
+            workflowId,
+            nodeId,
+            iteration: 0,
+            branch: item.branch,
+        })));
+        return {
+            state: aggregateNodeState(results.map((result) => result.state)),
+            tasks: results.flatMap((result) => result.tasks),
+        };
+    }
+    async runLoop(node, state, approvals, workflowId) {
+        const taskIds = [...state.taskIds];
+        const tasks = [];
+        let iterations = state.iterations ?? 0;
+        while (iterations < node.maxIterations) {
+            const offset = iterations * node.body.length;
+            for (const [index, request] of node.body.entries()) {
+                const result = await this.runTask(request, taskIds[offset + index], approvals, true, {
+                    workflowId,
+                    nodeId: node.id,
+                    iteration: iterations,
+                    branch: `loop:${index}`,
+                });
+                tasks.push(...result.tasks);
+                taskIds[offset + index] = result.state.taskIds[0];
+                if (result.state.status !== "verified_complete") {
+                    return {
+                        state: {
+                            ...result.state,
+                            taskIds,
+                            iterations,
+                        },
+                        tasks,
+                    };
+                }
+            }
+            iterations += 1;
+            const lastTaskId = taskIds[offset + node.body.length - 1];
+            if (node.until !== undefined &&
+                (await this.tasks.verifyPersisted(lastTaskId, [node.until])).verified) {
+                break;
+            }
+        }
+        return {
+            state: {
+                status: "verified_complete",
+                taskIds,
+                iterations,
+            },
+            tasks,
+        };
+    }
+    deriveStatus(run, definition) {
+        const required = definition.nodes.filter((node) => node.type !== "compensation");
+        if (required.some((node) => run.nodes[node.id]?.status === "failed")) {
+            return "failed";
+        }
+        if (required.some((node) => run.nodes[node.id]?.status === "recovery_required")) {
+            return "recovery_required";
+        }
+        if (required.some((node) => run.nodes[node.id]?.status === "awaiting_approval")) {
+            return "awaiting_approval";
+        }
+        if (required.every((node) => {
+            const state = run.nodes[node.id];
+            return (state !== undefined &&
+                ["verified_complete", "skipped", "compensated"].includes(state.status) &&
+                state.taskIds.every((taskId) => this.tasks.receipts({ taskId }).certificate?.result ===
+                    "VERIFIED_SUCCESS"));
+        })) {
+            return "verified_complete";
+        }
+        return "running";
+    }
+}
+function makeEvent(run, sequence, type, data) {
+    return WorkflowEventSchema.parse({
+        schemaVersion: "1.0.0",
+        id: randomUUID(),
+        aggregateId: run.id,
+        sequence,
+        traceId: run.traceId,
+        type,
+        data,
+        occurredAt: new Date().toISOString(),
+    });
+}
+function requestsFor(node) {
+    switch (node.type) {
+        case "operation":
+        case "compensation":
+            return [node.request];
+        case "condition":
+            return [...node.whenTrue, ...node.whenFalse];
+        case "parallel":
+            return node.branches.flat();
+        case "bounded_loop":
+            return node.body;
+        case "approval":
+        case "checkpoint":
+            return [];
+    }
+}
+function nodeStateForTask(task, plannedIsComplete) {
+    switch (task.status) {
+        case "verified_success":
+            return { status: "verified_complete", taskIds: [task.id] };
+        case "planned":
+            return {
+                status: plannedIsComplete ? "verified_complete" : "ready",
+                taskIds: [task.id],
+            };
+        case "awaiting_approval":
+            return {
+                status: "awaiting_approval",
+                taskIds: [task.id],
+                ...(task.approval === undefined ? {} : { approval: task.approval }),
+            };
+        case "recovery_required":
+            return {
+                status: "recovery_required",
+                taskIds: [task.id],
+                ...(task.error === undefined ? {} : { error: task.error }),
+            };
+        case "cancelled":
+            return { status: "cancelled", taskIds: [task.id] };
+        default:
+            return {
+                status: "failed",
+                taskIds: [task.id],
+                ...(task.error === undefined ? {} : { error: task.error }),
+            };
+    }
+}
+function aggregateNodeState(states) {
+    const taskIds = states.flatMap((state) => state.taskIds);
+    for (const status of [
+        "failed",
+        "recovery_required",
+        "awaiting_approval",
+        "cancelled",
+    ]) {
+        const state = states.find((candidate) => candidate.status === status);
+        if (state !== undefined)
+            return { ...state, taskIds };
+    }
+    return { status: "verified_complete", taskIds };
+}
+function taskIdempotencyKey(identity, request) {
+    return sha256({ ...identity, request });
+}
+function expectedTaskKeys(workflowId, node, definition) {
+    const key = (request, iteration, branch, nodeId = node.id) => taskIdempotencyKey({ workflowId, nodeId, iteration, branch }, request);
+    switch (node.type) {
+        case "operation":
+            return [key(node.request, 0, "operation")];
+        case "approval": {
+            const target = definition.nodes.find((candidate) => candidate.id === node.forNodeId);
+            return target?.type === "operation"
+                ? [key(target.request, 0, "operation", target.id)]
+                : [];
+        }
+        case "condition":
+            return [
+                ...node.whenTrue.map((request, index) => key(request, 0, `true:${index}`)),
+                ...node.whenFalse.map((request, index) => key(request, 0, `false:${index}`)),
+            ];
+        case "parallel":
+            return node.branches.flatMap((branch, branchIndex) => branch.map((request, requestIndex) => key(request, 0, `${branchIndex}:${requestIndex}`)));
+        case "bounded_loop":
+            return Array.from({ length: node.maxIterations }, (_, iteration) => node.body.map((request, index) => key(request, iteration, `loop:${index}`))).flat();
+        case "compensation":
+            return [key(node.request, 0, "compensation")];
+        case "checkpoint":
+            return [];
+    }
+}
+function preferredTask(tasks, idempotencyKey) {
+    return tasks
+        .filter((task) => task.idempotencyKey === idempotencyKey)
+        .sort((left, right) => Number(right.status === "verified_success") -
+        Number(left.status === "verified_success") ||
+        (right.attempt ?? 1) - (left.attempt ?? 1) ||
+        right.updatedAt.localeCompare(left.updatedAt))[0];
+}
+function recoveredNodeState(node, tasks, previous) {
+    if (tasks.length === 0)
+        return previous;
+    const taskIds = tasks.map((task) => task.id);
+    const withStatus = (status) => tasks.find((task) => task.status === status);
+    const recovery = withStatus("recovery_required");
+    if (recovery !== undefined) {
+        return {
+            status: "recovery_required",
+            taskIds,
+            ...(recovery.error === undefined ? {} : { error: recovery.error }),
+        };
+    }
+    const awaiting = withStatus("awaiting_approval");
+    if (awaiting !== undefined) {
+        return {
+            status: "awaiting_approval",
+            taskIds,
+            ...(awaiting.approval === undefined
+                ? {}
+                : { approval: awaiting.approval }),
+        };
+    }
+    const failed = tasks.find((task) => [
+        "failed",
+        "partial",
+        "budget_exhausted",
+        "policy_blocked",
+    ].includes(task.status));
+    if (failed !== undefined) {
+        return {
+            status: "failed",
+            taskIds,
+            ...(failed.error === undefined ? {} : { error: failed.error }),
+        };
+    }
+    if (tasks.some((task) => task.status === "planned")) {
+        return {
+            status: "pending",
+            taskIds,
+            ...(tasks.find((task) => task.error !== undefined)?.error === undefined
+                ? {}
+                : { error: tasks.find((task) => task.error !== undefined).error }),
+        };
+    }
+    if (tasks.every((task) => task.status === "verified_success")) {
+        return {
+            status: node.type === "compensation" ? "compensated" : "verified_complete",
+            taskIds,
+            ...(node.type === "bounded_loop"
+                ? {
+                    iterations: Math.ceil(tasks.length / node.body.length),
+                }
+                : {}),
+        };
+    }
+    return { status: "cancelled", taskIds };
+}
+//# sourceMappingURL=workflow-controller.js.map