@smithers-orchestrator/scheduler 0.17.0 → 0.19.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@smithers-orchestrator/scheduler",
-  "version": "0.17.0",
+  "version": "0.19.0",
   "description": "Pure decision engine: session, scheduler, and task state management for Smithers workflows",
   "type": "module",
   "sideEffects": false,
@@ -176,8 +176,8 @@
   ],
   "dependencies": {
     "effect": "^3.21.1",
-    "@smithers-orchestrator/errors": "0.17.0",
-    "@smithers-orchestrator/graph": "0.17.0"
+    "@smithers-orchestrator/errors": "0.19.0",
+    "@smithers-orchestrator/graph": "0.19.0"
   },
   "devDependencies": {
     "@types/bun": "latest",

package/src/ScheduleResult.ts

@@ -12,4 +12,6 @@ export type ScheduleResult = {
   readonly continuation?: ContinuationRequest;
   readonly nextRetryAtMs?: number;
   readonly fatalError?: string;
+  readonly failureRecoveryActive?: boolean;
+  readonly failureRecoveryKeys?: readonly string[];
 };

package/src/index.d.ts

@@ -74,6 +74,8 @@ type ScheduleResult$3 = {
     readonly continuation?: ContinuationRequest$1;
     readonly nextRetryAtMs?: number;
     readonly fatalError?: string;
+    readonly failureRecoveryActive?: boolean;
+    readonly failureRecoveryKeys?: readonly string[];
 };
 
 type ScheduleSnapshot$1 = {

package/src/makeWorkflowSession.js

@@ -179,8 +179,15 @@ function isRetryableFailure(descriptor, error) {
     const payloadCode = error && typeof error === "object" && typeof error.code === "string"
         ? error.code
         : undefined;
+    const payloadDetails = error && typeof error === "object" && error.details && typeof error.details === "object"
+        ? error.details
+        : undefined;
     const normalized = toSmithersError(error);
     const code = payloadCode ?? normalized.code;
+    const failureRetryable = payloadDetails?.failureRetryable ?? normalized.details?.failureRetryable;
+    if (failureRetryable === false || code === "AGENT_CONFIG_INVALID") {
+        return false;
+    }
     const isAgentTask = Boolean(descriptor.agent);
     const nonRetryableComputeCodes = new Set([
         "INVALID_OUTPUT",
@@ -372,6 +379,25 @@ export function makeWorkflowSession(options = {}) {
         state.failures.set(key, error);
         return decide();
     }
+    /**
+   * @returns {EngineDecision | null}
+   */
+    function unhandledFailureDecision(recoveryKeys = new Set()) {
+        for (const [key, taskState] of state.states) {
+            const parsed = parseStateKey(key);
+            const descriptor = findDescriptor(state, parsed.nodeId, parsed.iteration);
+            if (taskState === "failed" && !descriptor?.continueOnFail) {
+                if (recoveryKeys.has(key)) {
+                    continue;
+                }
+                return {
+                    _tag: "Failed",
+                    error: new SmithersError("SESSION_ERROR", `Task failed: ${descriptor?.nodeId ?? key}`, { key }, state.failures.get(key)),
+                };
+            }
+        }
+        return null;
+    }
     function ralphStatePayload() {
         return {
             ralphState: Object.fromEntries([...state.ralphState.entries()].map(([id, value]) => [
@@ -393,16 +419,6 @@ export function makeWorkflowSession(options = {}) {
         if (!state.graph) {
             return { _tag: "Wait", reason: { _tag: "ExternalTrigger" } };
         }
-        for (const [key, taskState] of state.states) {
-            const parsed = parseStateKey(key);
-            const descriptor = findDescriptor(state, parsed.nodeId, parsed.iteration);
-            if (taskState === "failed" && !descriptor?.continueOnFail) {
-                return {
-                    _tag: "Failed",
-                    error: new SmithersError("SESSION_ERROR", `Task failed: ${descriptor?.nodeId ?? key}`, { key }, state.failures.get(key)),
-                };
-            }
-        }
         const schedule = computeSchedule();
         if (schedule.fatalError) {
             return {
@@ -419,6 +435,11 @@ export function makeWorkflowSession(options = {}) {
                 },
             };
         }
+        const recoveryKeys = new Set(schedule.failureRecoveryKeys ?? []);
+        let failure = unhandledFailureDecision(recoveryKeys);
+        if (failure) {
+            return failure;
+        }
         const executable = [];
         let waitReason;
         let changed = false;
@@ -489,6 +510,10 @@ export function makeWorkflowSession(options = {}) {
         if ([...state.states.values()].some((taskState) => taskState === "in-progress")) {
             return { _tag: "Wait", reason: { _tag: "ExternalTrigger" } };
         }
+        failure = unhandledFailureDecision(recoveryKeys);
+        if (failure) {
+            return failure;
+        }
         if (schedule.readyRalphs.length > 0) {
             for (const ralph of schedule.readyRalphs) {
                 const current = state.ralphState.get(ralph.id) ?? {

package/src/scheduleTasks.js

@@ -70,6 +70,8 @@ export function scheduleTasks(plan, states, descriptors, ralphState, retryWait,
     let continuation;
     let nextRetryAtMs;
     let fatalError;
+    let failureRecoveryActive = false;
+    const failureRecoveryKeys = new Set();
     const groupUsage = new Map();
     for (const [stateKey, state] of states) {
         if (state !== "in-progress")
@@ -89,7 +91,7 @@ export function scheduleTasks(plan, states, descriptors, ralphState, retryWait,
    * @param {PlanNode} node
    * @returns {{ readonly terminal: boolean; readonly failed: boolean }}
    */
-    function inspect(node) {
+    function inspect(node, options = {}) {
         switch (node.kind) {
             case "task": {
                 const descriptor = descriptors.get(node.nodeId);
@@ -102,12 +104,16 @@ export function scheduleTasks(plan, states, descriptors, ralphState, retryWait,
                     state === "failed" ||
                     Boolean(descriptor.waitAsync &&
                         (state === "waiting-approval" || state === "waiting-event"));
-                return { terminal, failed: state === "failed" && !descriptor.continueOnFail };
+                return {
+                    terminal,
+                    failed: state === "failed" &&
+                        (options.includeContinuedFailures || !descriptor.continueOnFail),
+                };
             }
             case "sequence":
             case "group": {
                 for (const child of node.children) {
-                    const result = inspect(child);
+                    const result = inspect(child, options);
                     if (!result.terminal)
                         return { terminal: false, failed: false };
                     if (result.failed)
@@ -119,7 +125,7 @@ export function scheduleTasks(plan, states, descriptors, ralphState, retryWait,
                 let terminal = true;
                 let failed = false;
                 for (const child of node.children) {
-                    const result = inspect(child);
+                    const result = inspect(child, options);
                     if (!result.terminal)
                         terminal = false;
                     if (result.failed)
@@ -128,30 +134,137 @@ export function scheduleTasks(plan, states, descriptors, ralphState, retryWait,
                 return { terminal, failed: terminal && failed };
             }
             case "saga": {
+                let completedActions = 0;
+                let failed = false;
                 for (const child of node.actionChildren) {
-                    const result = inspect(child);
+                    const result = inspect(child, {
+                        includeContinuedFailures: true,
+                    });
+                    if (!result.terminal)
+                        return { terminal: false, failed: false };
+                    if (result.failed) {
+                        failed = true;
+                        break;
+                    }
+                    completedActions += 1;
+                }
+                if (!failed)
+                    return { terminal: true, failed: false };
+                if (node.onFailure === "fail")
+                    return { terminal: true, failed: true };
+                let compensationFailed = false;
+                for (let index = completedActions - 1; index >= 0; index -= 1) {
+                    const compensation = node.compensationChildren[index];
+                    if (!compensation)
+                        continue;
+                    const result = inspect(compensation, options);
                     if (!result.terminal)
                         return { terminal: false, failed: false };
                     if (result.failed)
-                        return { terminal: true, failed: true };
+                        compensationFailed = true;
                 }
-                return { terminal: true, failed: false };
+                return {
+                    terminal: true,
+                    failed: compensationFailed || node.onFailure === "compensate-and-fail",
+                };
             }
             case "try-catch-finally": {
+                let tryFailed = false;
                 for (const child of node.tryChildren) {
-                    const result = inspect(child);
+                    const result = inspect(child, {
+                        includeContinuedFailures: true,
+                    });
                     if (!result.terminal)
                         return { terminal: false, failed: false };
-                    if (result.failed)
-                        return { terminal: true, failed: true };
+                    if (result.failed) {
+                        tryFailed = true;
+                        break;
+                    }
                 }
-                return { terminal: true, failed: false };
+                if (!tryFailed) {
+                    return inspect({
+                        kind: "sequence",
+                        children: node.finallyChildren,
+                    }, options);
+                }
+                let catchFailed = node.catchChildren.length === 0;
+                if (node.catchChildren.length > 0) {
+                    const catchStatus = inspect({
+                        kind: "sequence",
+                        children: node.catchChildren,
+                    }, options);
+                    if (!catchStatus.terminal)
+                        return { terminal: false, failed: false };
+                    catchFailed = catchStatus.failed;
+                }
+                const finallyStatus = inspect({
+                    kind: "sequence",
+                    children: node.finallyChildren,
+                }, options);
+                if (!finallyStatus.terminal)
+                    return { terminal: false, failed: false };
+                return {
+                    terminal: true,
+                    failed: catchFailed || finallyStatus.failed,
+                };
             }
             default:
                 return { terminal: true, failed: false };
         }
     }
     /**
+   * @param {PlanNode} node
+   * @param {{ includeContinuedFailures?: boolean }} options
+   */
+    function collectFailureKeys(node, options = {}) {
+        switch (node.kind) {
+            case "task": {
+                const descriptor = descriptors.get(node.nodeId);
+                if (!descriptor)
+                    return;
+                const key = buildStateKey(descriptor.nodeId, descriptor.iteration);
+                const state = states.get(key) ?? "pending";
+                if (state === "failed" &&
+                    (options.includeContinuedFailures || !descriptor.continueOnFail)) {
+                    failureRecoveryKeys.add(key);
+                }
+                return;
+            }
+            case "sequence":
+            case "group":
+            case "parallel":
+                for (const child of node.children) {
+                    collectFailureKeys(child, options);
+                }
+                return;
+            case "saga":
+                for (const child of node.actionChildren) {
+                    collectFailureKeys(child, options);
+                }
+                return;
+            case "try-catch-finally":
+                for (const child of node.tryChildren) {
+                    collectFailureKeys(child, options);
+                }
+                for (const child of node.catchChildren) {
+                    collectFailureKeys(child, options);
+                }
+                for (const child of node.finallyChildren) {
+                    collectFailureKeys(child, options);
+                }
+                return;
+        }
+    }
+    /**
+   * @param {readonly PlanNode[]} children
+   * @param {{ includeContinuedFailures?: boolean }} options
+   */
+    function collectChildFailureKeys(children, options = {}) {
+        for (const child of children) {
+            collectFailureKeys(child, options);
+        }
+    }
+    /**
    * @param {readonly PlanNode[]} children
    */
     function walkSequence(children) {
@@ -247,7 +360,9 @@ export function scheduleTasks(plan, states, descriptors, ralphState, retryWait,
                 let completedActions = 0;
                 let failed = false;
                 for (const child of node.actionChildren) {
-                    const status = inspect(child);
+                    const status = inspect(child, {
+                        includeContinuedFailures: true,
+                    });
                     if (!status.terminal)
                         return walk(child);
                     if (status.failed) {
@@ -262,6 +377,23 @@ export function scheduleTasks(plan, states, descriptors, ralphState, retryWait,
                     fatalError ??= `Saga ${node.id} failed`;
                     return { terminal: true };
                 }
+                collectChildFailureKeys(node.actionChildren, {
+                    includeContinuedFailures: true,
+                });
+                let compensationFailed = false;
+                for (let index = completedActions - 1; index >= 0; index -= 1) {
+                    const compensation = node.compensationChildren[index];
+                    if (!compensation)
+                        continue;
+                    if (inspect(compensation).failed) {
+                        compensationFailed = true;
+                        break;
+                    }
+                }
+                if (compensationFailed) {
+                    return { terminal: false };
+                }
+                failureRecoveryActive = true;
                 for (let index = completedActions - 1; index >= 0; index -= 1) {
                     const compensation = node.compensationChildren[index];
                     if (!compensation)
@@ -278,7 +410,9 @@ export function scheduleTasks(plan, states, descriptors, ralphState, retryWait,
             case "try-catch-finally": {
                 let tryFailed = false;
                 for (const child of node.tryChildren) {
-                    const status = inspect(child);
+                    const status = inspect(child, {
+                        includeContinuedFailures: true,
+                    });
                     if (!status.terminal)
                         return walk(child);
                     if (status.failed) {
@@ -286,19 +420,72 @@ export function scheduleTasks(plan, states, descriptors, ralphState, retryWait,
                         break;
                     }
                 }
-                if (tryFailed) {
-                    if (node.catchChildren.length > 0) {
+                if (tryFailed && node.catchChildren.length > 0) {
+                    const collectTryFailureKeys = () => collectChildFailureKeys(node.tryChildren, {
+                        includeContinuedFailures: true,
+                    });
+                    let catchFailed = false;
+                    collectTryFailureKeys();
+                    const catchStatus = inspect({
+                        kind: "sequence",
+                        children: node.catchChildren,
+                    });
+                    failureRecoveryActive = true;
+                    catchFailed = catchStatus.failed;
+                    if (!catchStatus.terminal) {
                         const catchResult = walkSequence(node.catchChildren);
                         if (!catchResult.terminal)
                             return catchResult;
                     }
-                    else {
-                        fatalError ??= `TryCatchFinally ${node.id} failed`;
+                    const finallyStatus = inspect({
+                        kind: "sequence",
+                        children: node.finallyChildren,
+                    });
+                    if (finallyStatus.failed) {
+                        collectTryFailureKeys();
+                        failureRecoveryActive = false;
+                        return { terminal: false };
                     }
+                    const finallyResult = walkSequence(node.finallyChildren);
+                    if (!finallyResult.terminal) {
+                        collectTryFailureKeys();
+                        if (catchFailed) {
+                            collectChildFailureKeys(node.catchChildren);
+                        }
+                        failureRecoveryActive = true;
+                        return finallyResult;
+                    }
+                    if (catchFailed) {
+                        return { terminal: true };
+                    }
+                    return { terminal: true };
+                }
+                const finallyStatus = inspect({
+                    kind: "sequence",
+                    children: node.finallyChildren,
+                });
+                if (finallyStatus.failed) {
+                    if (tryFailed) {
+                        collectChildFailureKeys(node.tryChildren, {
+                            includeContinuedFailures: true,
+                        });
+                    }
+                    failureRecoveryActive = false;
+                    return { terminal: false };
                 }
                 const finallyResult = walkSequence(node.finallyChildren);
-                if (!finallyResult.terminal)
+                if (!finallyResult.terminal) {
+                    if (tryFailed && node.catchChildren.length === 0) {
+                        collectChildFailureKeys(node.tryChildren, {
+                            includeContinuedFailures: true,
+                        });
+                        failureRecoveryActive = true;
+                    }
                     return finallyResult;
+                }
+                if (tryFailed && node.catchChildren.length === 0) {
+                    fatalError ??= `TryCatchFinally ${node.id} failed`;
+                }
                 return { terminal: true };
             }
             case "group": {
@@ -326,5 +513,7 @@ export function scheduleTasks(plan, states, descriptors, ralphState, retryWait,
         continuation,
         nextRetryAtMs,
         fatalError,
+        failureRecoveryActive,
+        failureRecoveryKeys: [...failureRecoveryKeys],
     };
 }