@pikku/core 0.12.134 → 0.12.136

package/dist/wirings/workflow/graph/graph-runner.js

@@ -1,5 +1,6 @@
 import { WorkflowAsyncException, WorkflowSuspendedException, } from '../workflow-errors.js';
-import { DEFAULT_STEP_RETRIES } from '../workflow-constants.js';
+import { DEFAULT_STEP_RETRIES, NO_COMPENSATE_STATE_PREFIX, } from '../workflow-constants.js';
+import { childRunFailure } from '../workflow-compensation.js';
 import { runningStepLease } from '../workflow-step-lease.js';
 import { pikkuState, getSingletonServices } from '../../../pikku-state.js';
 import { RPCNotFoundError } from '../../rpc/rpc-runner.js';
@@ -87,7 +88,7 @@ function closesCycle(source, target, nodes) {
     }
     return false;
 }
-function planGraphTransitions(nodes, instances, branchByStep, entryNodeIds, graphName, fanoutWidths = {}) {
+function planGraphTransitions(nodes, instances, branchByStep, entryNodeIds, graphName, fanoutWidths = {}, recovered = new Set()) {
     const toLogical = (name) => remapStepNamesToNodeIds([name], nodes, graphName)[0];
     const countByLogical = {};
     const instancesByLogical = {};
@@ -103,10 +104,12 @@ function planGraphTransitions(nodes, instances, branchByStep, entryNodeIds, grap
         const width = fanoutWidths[nodeId];
         if (width === undefined)
             return false;
-        const succeeded = (instancesByLogical[nodeId] ?? []).filter((i) => i.status === 'succeeded').length;
+        const succeeded = (instancesByLogical[nodeId] ?? []).filter(isDone).length;
         return succeeded >= width;
     };
-    const completed = instances.filter((i) => i.status === 'succeeded');
+    const ignored = (name) => recovered.has(name) && nodes[toLogical(name)]?.recover === 'ignore';
+    const isDone = (i) => i.status === 'succeeded' || ignored(i.stepName);
+    const completed = instances.filter(isDone);
     const completedLogical = new Set();
     for (const inst of completed) {
         const logical = toLogical(inst.stepName);
@@ -121,6 +124,20 @@ function planGraphTransitions(nodes, instances, branchByStep, entryNodeIds, grap
     for (const entryId of entryNodeIds) {
         edges.push({ fromKey: ENTRY_FROM, target: entryId });
     }
+    for (const inst of instances) {
+        if (!recovered.has(inst.stepName))
+            continue;
+        const fromLogical = toLogical(inst.stepName);
+        for (const target of recoverTargets(nodes[fromLogical])) {
+            edges.push({
+                from: inst.stepName,
+                fromKey: inst.stepName,
+                fromLogical,
+                target,
+                recoveringFrom: { nodeId: fromLogical, stepName: inst.stepName },
+            });
+        }
+    }
     const fannedEdgesEmitted = new Set();
     for (const inst of completed) {
         const fromLogical = toLogical(inst.stepName);
@@ -155,12 +172,23 @@ function planGraphTransitions(nodes, instances, branchByStep, entryNodeIds, grap
             edges.push({ from: nodeId, fromKey: nodeId, fromLogical: nodeId, target });
         }
     }
+    const deadLogical = new Set(instances
+        .filter((i) => recovered.has(i.stepName) && !isDone(i))
+        .map((i) => toLogical(i.stepName)));
+    const readsDeadNode = (nodeId) => nodeDependencies(nodes[nodeId] ?? {}).some((dep) => deadLogical.has(dep));
     const toFire = [];
     const plannedFanout = new Set();
     let blockedWaiting = false;
     for (const edge of edges) {
         const target = edge.target;
         const edgeKey = `${edge.fromKey}->${target}`;
+        // A node reading the output of a failure that was recovered without
+        // `'ignore'` has no output to read and never will; waiting on it would
+        // leave the run unable to finish.
+        if (readsDeadNode(target)) {
+            consumed.add(edgeKey);
+            continue;
+        }
         if (isFanned(target)) {
             if (plannedFanout.has(target))
                 continue;
@@ -221,13 +249,14 @@ function planGraphTransitions(nodes, instances, branchByStep, entryNodeIds, grap
             logical: target,
             instanceKey: visits === 0 ? target : `${target}#${visits}`,
             fromStepName: edge.from,
+            recoveringFrom: edge.recoveringFrom,
         });
         countByLogical[target] = visits + 1;
         consumed.add(edgeKey);
     }
     return {
         toFire,
-        hasInFlight: instances.some((i) => i.status !== 'succeeded'),
+        hasInFlight: instances.some((i) => !isDone(i) && !recovered.has(i.stepName)),
         blockedWaiting,
     };
 }
@@ -396,6 +425,8 @@ function extractReferencedNodeIds(input) {
 }
 const IGNORED_REFS = new Set(['trigger', '$item', 'unknown']);
 const FOREACH_MODES = new Set(['parallel', 'sequential']);
+const recoverTargets = (node) => node.recover === 'ignore' ? [] : normalizeNodeTargets(node.recover);
+const recoversFailure = (node) => !!node?.recover;
 function normalizeNodeTargets(value) {
     if (!value)
         return [];
@@ -447,10 +478,9 @@ function validateGraphReferences(graphName, nodes, entryNodes) {
                 throw new Error(`Workflow graph '${graphName}': node '${nodeId}' routes to unknown node '${nextId}'`);
             }
         }
-        const errorTargets = normalizeNodeTargets(node.onError);
-        for (const errorId of errorTargets) {
+        for (const errorId of recoverTargets(node)) {
             if (!nodeIds.has(errorId)) {
-                throw new Error(`Workflow graph '${graphName}': node '${nodeId}' onError targets unknown node '${errorId}'`);
+                throw new Error(`Workflow graph '${graphName}': node '${nodeId}' recover targets unknown node '${errorId}'`);
             }
         }
     }
@@ -564,6 +594,51 @@ async function createGraphResultReader(workflowService, runId, graphName, nodes,
         },
     };
 }
+const splitFailures = (rawFailed, nodes, graphName) => {
+    const logical = remapStepNamesToNodeIds(rawFailed, nodes, graphName);
+    const recovered = new Set();
+    const unrecovered = [];
+    rawFailed.forEach((stepName, i) => {
+        if (recoversFailure(nodes[logical[i]]))
+            recovered.add(stepName);
+        else
+            unrecovered.push(logical[i]);
+    });
+    return { recovered, unrecovered };
+};
+async function failGraphRun(workflowService, runId, failedNode, rpcService) {
+    await failGraphRunWith(workflowService, runId, new Error(`Graph node '${failedNode}' failed after exhausting retries`), rpcService);
+}
+async function failGraphRunWith(workflowService, runId, cause, rpcService) {
+    const error = Object.assign(cause, { code: 'GRAPH_NODE_FAILED' });
+    const run = await workflowService.getRun(runId);
+    if (run && rpcService) {
+        await workflowService.failOrUnwind(run, error, rpcService);
+        return;
+    }
+    await workflowService.updateRunStatus(runId, 'failed', undefined, {
+        message: error.message,
+        stack: error.stack ?? '',
+        code: 'GRAPH_NODE_FAILED',
+    });
+    if (run)
+        await workflowService.notifyChildWorkflowFailed(run, error);
+}
+const RECOVERING_FROM_STATE_PREFIX = 'recoveringFrom:';
+async function prepareFire(workflowService, runId, fire, node) {
+    if (node.compensate === false) {
+        await workflowService.updateRunState(runId, `${NO_COMPENSATE_STATE_PREFIX}${fire.instanceKey}`, true);
+    }
+    const from = fire.recoveringFrom;
+    if (!from) {
+        await workflowService.updateRunState(runId, `${RECOVERING_FROM_STATE_PREFIX}${fire.instanceKey}`, null);
+        return;
+    }
+    const steps = await workflowService.getRunSteps(runId);
+    const failed = steps.find((step) => step.stepName === from.stepName);
+    await workflowService.updateRunState(runId, `${RECOVERING_FROM_STATE_PREFIX}${fire.instanceKey}`, { ...from, error: failed?.error ?? { message: 'step failed' } });
+    await workflowService.updateRunState(runId, `${NO_COMPENSATE_STATE_PREFIX}${from.stepName}`, true);
+}
 const graphStepOptions = (nodeConfig) => ({
     retries: nodeConfig?.retries ?? DEFAULT_STEP_RETRIES,
     retryDelay: nodeConfig?.retryDelay,
@@ -573,7 +648,7 @@ async function queueGraphNode(workflowService, runId, _graphName, nodeId, rpcNam
     await workflowService.insertStepState(runId, nodeId, rpcName, input, stepOptions, fromStepName);
     await workflowService.queueStepWorker(runId, nodeId, rpcName, input, stepOptions, fromStepName);
 }
-export async function continueGraph(workflowService, runId, graphName, overrideMeta) {
+export async function continueGraph(workflowService, runId, graphName, overrideMeta, rpcService) {
     const meta = overrideMeta ?? getWorkflowMeta(graphName);
     if (!meta?.nodes) {
         throw new Error(`Workflow graph meta '${graphName}' not found`);
@@ -583,14 +658,9 @@ export async function continueGraph(workflowService, runId, graphName, overrideM
     const { completedNodeIds: rawCompleted, failedNodeIds: rawFailed, branchKeys: branchByStep, } = await workflowService.getCompletedGraphState(runId);
     remapStepNamesToNodeIds(rawCompleted, nodes, graphName);
     remapBranchKeys(branchByStep, nodes, graphName);
-    const failedNodeIds = remapStepNamesToNodeIds(rawFailed, nodes, graphName);
-    if (failedNodeIds.length > 0) {
-        const failedNode = failedNodeIds[0];
-        await workflowService.updateRunStatus(runId, 'failed', undefined, {
-            message: `Graph node '${failedNode}' failed after exhausting retries`,
-            stack: '',
-            code: 'GRAPH_NODE_FAILED',
-        });
+    const { recovered, unrecovered } = splitFailures(rawFailed, nodes, graphName);
+    if (unrecovered.length > 0) {
+        await failGraphRun(workflowService, runId, unrecovered[0], rpcService);
         return;
     }
     const currentRun = await workflowService.getRun(runId);
@@ -600,7 +670,7 @@ export async function continueGraph(workflowService, runId, graphName, overrideM
     const instances = await workflowService.getStepInstances(runId);
     const triggerInput = currentRun?.input;
     const reader = await createGraphResultReader(workflowService, runId, graphName, nodes, triggerInput, instances);
-    const plan = planGraphTransitions(nodes, instances, branchByStep, meta.entryNodeIds ?? [], graphName, reader.fanoutWidths);
+    const plan = planGraphTransitions(nodes, instances, branchByStep, meta.entryNodeIds ?? [], graphName, reader.fanoutWidths, recovered);
     const resolveNodeInput = async (logical, itemIndex) => {
         const node = nodes[logical];
         const referencedNodeIds = extractReferencedNodeIds(node.input).filter((id) => !IGNORED_REFS.has(id));
@@ -638,6 +708,7 @@ export async function continueGraph(workflowService, runId, graphName, overrideM
         if (!node?.rpcName)
             continue;
         const resolvedInput = await resolveNodeInput(fire.logical, fire.itemIndex);
+        await prepareFire(workflowService, runId, fire, node);
         await queueGraphNode(workflowService, runId, graphName, fire.instanceKey, node.rpcName, resolvedInput, node, fire.fromStepName);
     }
 }
@@ -652,6 +723,7 @@ async function invokeGraphNodeRpc(workflowService, rpcService, runId, stepId, no
         },
         setState: (name, value) => workflowService.updateRunState(runId, name, value),
         getState: () => workflowService.getRunState(runId),
+        recoveringFrom: (await workflowService.getRunState(runId))[`${RECOVERING_FROM_STATE_PREFIX}${nodeId}`],
     };
     const result = await rpcService.rpcWithWire(rpcName, input, {
         graph: graphWire,
@@ -681,12 +753,9 @@ export async function executeGraphStep(workflowService, rpcService, runId, stepI
             await workflowService.setStepChildRunId(stepId, childRunId);
             if (shouldInline) {
                 const childRun = await workflowService.getRun(childRunId);
-                if (childRun?.status === 'failed') {
-                    throw new Error(childRun.error?.message || 'Sub-workflow failed');
-                }
-                if (childRun?.status === 'cancelled') {
-                    throw new Error('Sub-workflow was cancelled');
-                }
+                const failure = childRun && childRunFailure(childRun);
+                if (failure)
+                    throw failure;
                 result = childRun?.output;
             }
             else {
@@ -717,27 +786,11 @@ export async function executeGraphStep(workflowService, rpcService, runId, stepI
             });
             throw error;
         }
-        const meta = getWorkflowMeta(graphName);
-        if (meta?.nodes) {
-            const node = meta.nodes[nodeId];
-            if (node?.onError) {
-                const errorNodes = Array.isArray(node.onError)
-                    ? node.onError
-                    : [node.onError];
-                for (const errorNodeId of errorNodes) {
-                    const errorNode = meta.nodes[errorNodeId];
-                    if (errorNode) {
-                        await queueGraphNode(workflowService, runId, graphName, errorNodeId, errorNode.rpcName, { error: { message: error.message } }, errorNode);
-                    }
-                }
-                throw error;
-            }
-        }
         throw error;
     }
 }
 export async function runFromMeta(workflowService, runId, meta, _rpcService) {
-    await continueGraph(workflowService, runId, meta.name, meta);
+    await continueGraph(workflowService, runId, meta.name, meta, _rpcService);
 }
 async function executeGraphNodeInline(workflowService, rpcService, runId, graphName, nodeId, instanceKey, input, nodes, fromStepName) {
     const node = nodes[nodeId];
@@ -762,12 +815,9 @@ async function executeGraphNodeInline(workflowService, rpcService, runId, graphN
             const { runId: childRunId } = await workflowService.startWorkflow(rpcName, input, childWire, rpcService, { inline: true });
             await workflowService.setStepChildRunId(stepState.stepId, childRunId);
             const childRun = await workflowService.getRun(childRunId);
-            if (childRun?.status === 'failed') {
-                throw new Error(childRun.error?.message || 'Sub-workflow failed');
-            }
-            if (childRun?.status === 'cancelled') {
-                throw new Error('Sub-workflow was cancelled');
-            }
+            const failure = childRun && childRunFailure(childRun);
+            if (failure)
+                throw failure;
             result = childRun?.output;
         }
         else if (agentMeta) {
@@ -793,13 +843,8 @@ async function executeGraphNodeInline(workflowService, rpcService, runId, graphN
             throw new WorkflowSuspendedException(runId, 'RPC_NOT_FOUND');
         }
         await workflowService.setStepError(stepState.stepId, error);
-        if (node?.onError) {
-            const errorNodes = Array.isArray(node.onError)
-                ? node.onError
-                : [node.onError];
-            await Promise.all(errorNodes.map((errorNodeId) => executeGraphNodeInline(workflowService, rpcService, runId, graphName, errorNodeId, errorNodeId, { error: { message: error.message } }, nodes, nodeId)));
+        if (recoversFailure(node))
             return;
-        }
         throw error;
     }
 }
@@ -808,14 +853,9 @@ async function continueGraphInline(workflowService, rpcService, runId, graphName
         const { failedNodeIds: rawFailed, branchKeys: branchByStep, completedNodeIds: rawCompleted, } = await workflowService.getCompletedGraphState(runId);
         remapStepNamesToNodeIds(rawCompleted, nodes, graphName);
         remapBranchKeys(branchByStep, nodes, graphName);
-        const failedNodeIds = remapStepNamesToNodeIds(rawFailed, nodes, graphName);
-        if (failedNodeIds.length > 0) {
-            const failedNode = failedNodeIds[0];
-            await workflowService.updateRunStatus(runId, 'failed', undefined, {
-                message: `Graph node '${failedNode}' failed after exhausting retries`,
-                stack: '',
-                code: 'GRAPH_NODE_FAILED',
-            });
+        const { recovered, unrecovered } = splitFailures(rawFailed, nodes, graphName);
+        if (unrecovered.length > 0) {
+            await failGraphRun(workflowService, runId, unrecovered[0], rpcService);
             return;
         }
         const run = await workflowService.getRun(runId);
@@ -824,7 +864,7 @@ async function continueGraphInline(workflowService, rpcService, runId, graphName
         }
         const instances = await workflowService.getStepInstances(runId);
         const reader = await createGraphResultReader(workflowService, runId, graphName, nodes, triggerInput, instances);
-        const plan = planGraphTransitions(nodes, instances, branchByStep, entryNodeIds, graphName, reader.fanoutWidths);
+        const plan = planGraphTransitions(nodes, instances, branchByStep, entryNodeIds, graphName, reader.fanoutWidths, recovered);
         if (plan.toFire.length === 0) {
             if (!plan.hasInFlight && !plan.blockedWaiting) {
                 await workflowService.updateRunStatus(runId, 'completed');
@@ -848,6 +888,7 @@ async function continueGraphInline(workflowService, rpcService, runId, graphName
             }
             const resolvedInput = resolveSerializedInput(node.input, nodeResults);
             executed++;
+            await prepareFire(workflowService, runId, fire, node);
             await executeGraphNodeInline(workflowService, rpcService, runId, graphName, fire.logical, fire.instanceKey, resolvedInput, nodes, fire.fromStepName);
         }));
         if (executed === 0)
@@ -926,11 +967,7 @@ export async function runWorkflowGraph(workflowService, graphName, triggerInput,
                     error instanceof WorkflowSuspendedException) {
                     return;
                 }
-                await workflowService.updateRunStatus(runId, 'failed', undefined, {
-                    message: error.message,
-                    stack: error.stack || '',
-                    code: 'GRAPH_NODE_FAILED',
-                });
+                await failGraphRunWith(workflowService, runId, error, rpcService);
             }
             finally {
                 workflowService.unregisterInlineRun(runId);

package/dist/wirings/workflow/graph/graph-validation.js

@@ -24,9 +24,9 @@ export function generateMermaidDiagram(workflowName, nodes, entryNodeIds) {
         for (const target of nextTargets) {
             lines.push(`  ${nodeId} --> ${target}`);
         }
-        const errorTargets = normalizeTargets(node.onError);
+        const errorTargets = node.recover === 'ignore' ? [] : normalizeTargets(node.recover);
         for (const target of errorTargets) {
-            lines.push(`  ${nodeId} -.->|error| ${target}`);
+            lines.push(`  ${nodeId} -.->|recover| ${target}`);
         }
     }
     return lines.join('\n');

package/dist/wirings/workflow/graph/workflow-graph.types.d.ts

@@ -26,7 +26,14 @@ export interface GraphNodeConfig<NodeIds extends string = string> {
     mode?: ForEachMode;
     input?: (ref: RefFn<NodeIds>, template: TemplateFn, $item: ItemFn) => Record<string, unknown>;
     next?: NextConfig<NodeIds>;
-    onError?: NodeIds | NodeIds[];
+    /**
+     * What happens when this node fails after its retries: run these nodes
+     * instead (the error is on `graph.recoveringFrom`), or `'ignore'` to carry
+     * on to `next` with a null output. A recovered failure is not compensated.
+     */
+    recover?: NodeIds | NodeIds[] | 'ignore';
+    /** Leave this node out of the saga unwind. */
+    compensate?: false;
     retries?: number;
     retryDelay?: string | number;
     notes?: string;
@@ -38,6 +45,18 @@ export interface PikkuGraphWire {
     branch: (key: string) => void;
     setState: (name: string, value: unknown) => Promise<void>;
     getState: () => Promise<Record<string, unknown>>;
+    /** Set on a node that runs because another node failed and named it in `recover`. */
+    recoveringFrom?: GraphRecovery;
+}
+export interface GraphRecovery {
+    nodeId: string;
+    stepName: string;
+    error: {
+        message: string;
+        stack?: string;
+        code?: string;
+        name?: string;
+    };
 }
 export interface GraphWireState {
     branchKey?: string;

package/dist/wirings/workflow/index.d.ts

@@ -16,6 +16,6 @@ export { template, type TemplateString } from './graph/template.js';
 export { pikkuWorkflowGraph, type PikkuWorkflowGraphConfig, type PikkuWorkflowGraphResult, } from './graph/wire-workflow-graph.js';
 export type { WorkflowStepInput as WorkflowStepQueueInput, PikkuWorkflowOrchestratorInput, PikkuWorkflowSleeperInput, } from './workflow-queue-workers.js';
 export type { WorkflowService, WorkflowQueueOptions, WorkflowServiceOptions, WorkflowServiceConfig, WorkflowPlannedStep, WorkflowRunWire, WorkflowStatus, WorkflowVersionStatus, StepStatus, WorkflowRun, WorkflowRunStatus, StepState, WorkflowRunService, WorkflowRunMirror, CoreWorkflow, PikkuWorkflow, ContextVariable, WorkflowContext, WorkflowsMeta, WorkflowsRuntimeMeta, } from './workflow.types.js';
-export type { WorkflowStepOptions, WorkflowWireDoRPC, WorkflowApprovalOptions, ApprovalOutcome, InputSource, OutputBinding, RpcStepMeta, Condition, BranchStepMeta, ParallelGroupStepMeta, FanoutStepMeta, ReturnStepMeta, InlineStepMeta, SleepStepMeta, CancelStepMeta, SuspendStepMeta, ApprovalStepMeta, SetStepMeta, SwitchCaseMeta, SwitchStepMeta, FilterStepMeta, ArrayPredicateStepMeta, WorkflowStepMeta, WorkflowStepWire, PikkuWorkflowWire, } from './workflow.types.js';
+export type { WorkflowStepOptions, WorkflowWireDoRPC, WorkflowApprovalOptions, ApprovalOutcome, InputSource, OutputBinding, RpcStepMeta, Condition, BranchStepMeta, ParallelGroupStepMeta, FanoutStepMeta, ReturnStepMeta, InlineStepMeta, SleepStepMeta, CancelStepMeta, SuspendStepMeta, MilestoneStepMeta, ApprovalStepMeta, SetStepMeta, SwitchCaseMeta, SwitchStepMeta, FilterStepMeta, ArrayPredicateStepMeta, WorkflowStepMeta, WorkflowStepWire, PikkuWorkflowWire, } from './workflow.types.js';
 export { createGraph } from './graph/graph-node.js';
 export type { GraphNodeConfig, ForEachConfig, ForEachMode, ItemFn, TemplateFn, } from './graph/workflow-graph.types.js';

package/dist/wirings/workflow/pikku-workflow-service.d.ts

@@ -52,6 +52,11 @@ export declare abstract class PikkuWorkflowService implements WorkflowService {
     abstract getRunHistory(runId: string): Promise<Array<StepState & {
         stepName: string;
     }>>;
+    abstract getRunSteps(runId: string): Promise<Array<StepState & {
+        stepName: string;
+        rpcName?: string;
+        data?: any;
+    }>>;
     updateRunStatus(id: string, status: WorkflowStatus, output?: any, error?: SerializedError): Promise<void>;
     protected abstract updateRunStatusImpl(id: string, status: WorkflowStatus, output?: any, error?: SerializedError): Promise<void>;
     insertStepState(runId: string, stepName: string, rpcName: string | null, data: any, stepOptions?: WorkflowStepOptions, fromStepName?: string): Promise<StepState>;
@@ -212,6 +217,7 @@ export declare abstract class PikkuWorkflowService implements WorkflowService {
     runWorkflowJob(runId: string, rpcService: PikkuRPC): Promise<void>;
     private runWorkflowJobInner;
     private onChildWorkflowCompleted;
+    notifyChildWorkflowFailed(childRun: WorkflowRun, error: Error): Promise<void>;
     protected onChildWorkflowFailed(childRun: WorkflowRun, error: Error): Promise<void>;
     executeWorkflowStep(runId: string, stepName: string, rpcName: string, data: any, rpcService: PikkuRPC): Promise<void>;
     /**
@@ -245,6 +251,13 @@ export declare abstract class PikkuWorkflowService implements WorkflowService {
     approveStep(runId: string, reason: string, decision: unknown, session?: CoreUserSession): Promise<void>;
     private approvalStep;
     createWorkflowWire(name: string, runId: string, rpcService: PikkuRPC, addonNamespace?: string | null): PikkuWorkflowWire;
+    private compensationApi;
+    failOrUnwind(run: WorkflowRun, error: Error & {
+        code?: string;
+    }, rpcService: PikkuRPC): Promise<'unwinding' | 'finished' | 'failed'>;
+    /** Stops an unfinished run, children first, and unwinds what it completed. */
+    cancelRun(runId: string, rpcService: PikkuRPC, reason?: string): Promise<WorkflowStatus | undefined>;
+    protected markFailedUnlessUnwound(runId: string, error: SerializedError): Promise<void>;
     protected verifyStepName(stepName: string): void;
     private getConfig;
     protected getOrchestratorQueueName(workflowName?: string): string;