@pikku/core 0.12.133 → 0.12.135
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/dist/function/compensation-name.d.ts +4 -0
- package/dist/function/compensation-name.js +6 -0
- package/dist/function/function-meta.types.d.ts +2 -0
- package/dist/function/function-runner.js +17 -0
- package/dist/function/functions.types.d.ts +7 -0
- package/dist/testing/service-tests/queued-workflow-harness.d.ts +61 -0
- package/dist/testing/service-tests/queued-workflow-harness.js +151 -0
- package/dist/testing/service-tests/workflow-compensation-queued-tests.d.ts +11 -0
- package/dist/testing/service-tests/workflow-compensation-queued-tests.js +683 -0
- package/dist/testing/service-tests.d.ts +4 -0
- package/dist/testing/service-tests.js +4 -0
- package/dist/utils/hmac.d.ts +0 -33
- package/dist/utils/hmac.js +0 -61
- package/dist/wirings/rpc/rpc-runner.js +21 -1
- package/dist/wirings/trigger/webhook-source-runner.js +17 -10
- package/dist/wirings/trigger/webhook-source.types.d.ts +4 -3
- package/dist/wirings/workflow/dsl/workflow-dsl.types.d.ts +38 -6
- package/dist/wirings/workflow/graph/graph-node.d.ts +2 -1
- package/dist/wirings/workflow/graph/graph-node.js +2 -1
- package/dist/wirings/workflow/graph/graph-runner.d.ts +1 -1
- package/dist/wirings/workflow/graph/graph-runner.js +104 -67
- package/dist/wirings/workflow/graph/graph-validation.js +2 -2
- package/dist/wirings/workflow/graph/workflow-graph.types.d.ts +20 -1
- package/dist/wirings/workflow/index.d.ts +1 -1
- package/dist/wirings/workflow/pikku-workflow-service.d.ts +13 -0
- package/dist/wirings/workflow/pikku-workflow-service.js +113 -143
- package/dist/wirings/workflow/run-timeline.d.ts +2 -0
- package/dist/wirings/workflow/run-timeline.js +3 -0
- package/dist/wirings/workflow/workflow-child-step.d.ts +17 -0
- package/dist/wirings/workflow/workflow-child-step.js +31 -0
- package/dist/wirings/workflow/workflow-compensation.d.ts +68 -0
- package/dist/wirings/workflow/workflow-compensation.js +282 -0
- package/dist/wirings/workflow/workflow-constants.d.ts +5 -0
- package/dist/wirings/workflow/workflow-constants.js +9 -0
- package/dist/wirings/workflow/workflow-dsl-pass.d.ts +12 -0
- package/dist/wirings/workflow/workflow-dsl-pass.js +75 -0
- package/dist/wirings/workflow/workflow-queue-routing.js +3 -3
- package/dist/wirings/workflow/workflow-run-status.js +13 -1
- package/dist/wirings/workflow/workflow-status-stream.js +2 -0
- package/dist/wirings/workflow/workflow-unwind-plan.d.ts +38 -0
- package/dist/wirings/workflow/workflow-unwind-plan.js +120 -0
- package/dist/wirings/workflow/workflow.types.d.ts +9 -2
- package/knowledge/decisions/internals/workflow-step-compensation-runs-as-its-own-durable-step.md +21 -18
- package/package.json +1 -1
- package/src/public-surface.json +0 -1
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { getSingletonServices, getCreateWireServices, pikkuState, } from '../../pikku-state.js';
|
|
1
|
+
import { getSingletonServices, pikkuState } from '../../pikku-state.js';
|
|
3
2
|
import { getDurationInMilliseconds } from '../../time-utils.js';
|
|
4
3
|
import { ChildWorkflowStartedException, continueGraph, executeGraphStep, runWorkflowGraph, stripInstanceOrdinal, } from './graph/graph-runner.js';
|
|
5
4
|
import { isExpectedError } from '../../errors/error-handler.js';
|
|
@@ -8,8 +7,8 @@ import { RPCNotFoundError } from '../rpc/rpc-runner.js';
|
|
|
8
7
|
import { deriveInvocationId } from './workflow-invocation-id.js';
|
|
9
8
|
import { approvalDeciderFrom } from './workflow-approval-policy.js';
|
|
10
9
|
import { buildRunTimeline, reconstructStateAt, } from './run-timeline.js';
|
|
11
|
-
import { DEFAULT_STEP_RETRIES, RUN_LEASE_RETRY_MS, WORKFLOW_CHILD_POLL_MAX_MS, WORKFLOW_END_STATES, WORKFLOW_POLL_FACTOR, WORKFLOW_POLL_MIN_MS, WORKFLOW_TERMINAL_STATES, isRunSettled, } from './workflow-constants.js';
|
|
12
|
-
import { WorkflowAsyncException,
|
|
10
|
+
import { DEFAULT_STEP_RETRIES, RUN_LEASE_RETRY_MS, WORKFLOW_CHILD_POLL_MAX_MS, WORKFLOW_END_STATES, WORKFLOW_POLL_FACTOR, WORKFLOW_POLL_MIN_MS, WORKFLOW_TERMINAL_STATES, NO_COMPENSATE_STATE_PREFIX, forwardStepName, isCompensationStepName, milestoneStepName, isRunSettled, } from './workflow-constants.js';
|
|
11
|
+
import { WorkflowAsyncException, WorkflowDispatchException, WorkflowNotFoundError, WorkflowRunCancelledError, WorkflowRunFailedError, WorkflowRunNotFoundError, WorkflowStepNameNotString, WorkflowStepSupersededError, } from './workflow-errors.js';
|
|
13
12
|
import { resolveWorkflowMeta } from './workflow-meta-resolver.js';
|
|
14
13
|
import { storeWorkflowVersion } from './workflow-version-store.js';
|
|
15
14
|
import { jobGroupFor, orchestratorQueueName, requireQueueService, resolveWorkflowConfig, stepDispatchTarget, stepJobOptions, stepWorkerQueueName, } from './workflow-queue-routing.js';
|
|
@@ -22,6 +21,9 @@ import { holdWorkflowLease, isRunLeaseError, nullWhenStepHeld, } from './workflo
|
|
|
22
21
|
import { summarizeRunStatus } from './workflow-run-status.js';
|
|
23
22
|
import { runningStepLease, startStepLeaseRefresh, stepLeaseMsForQueue, } from './workflow-step-lease.js';
|
|
24
23
|
import { runInlineRetryLoop } from './workflow-step-retry.js';
|
|
24
|
+
import { runInlineChildStep, startQueuedChildStep, } from './workflow-child-step.js';
|
|
25
|
+
import { runDslWorkflowPass } from './workflow-dsl-pass.js';
|
|
26
|
+
import { childRunFailure, failRunOrUnwind, isUnwound, compensatingForStep, cancelAndUnwind, createUnwindHost, driveUnwind, recordHasCompensation, isChildWorkflowRecord, } from './workflow-compensation.js';
|
|
25
27
|
import { runVersionMismatchFallback } from './workflow-version-fallback.js';
|
|
26
28
|
import { RedispatchBackoff, sweepStalledRuns, sweepUndispatchedSteps, } from './workflow-recovery.js';
|
|
27
29
|
export class PikkuWorkflowService {
|
|
@@ -426,12 +428,16 @@ export class PikkuWorkflowService {
|
|
|
426
428
|
error.name !== 'WorkflowSuspendedException' &&
|
|
427
429
|
error.name !== 'WorkflowDispatchException' &&
|
|
428
430
|
!isRunLeaseError(error)) {
|
|
429
|
-
await this.
|
|
431
|
+
await this.markFailedUnlessUnwound(runId, {
|
|
430
432
|
name: error.name,
|
|
431
433
|
message: error.message,
|
|
432
434
|
stack: error.stack,
|
|
433
435
|
});
|
|
434
436
|
getSingletonServices().logger.error(`Workflow ${name} (run ${runId}) failed:`, isExpectedError(error) ? error.message : error);
|
|
437
|
+
const settled = await this.getRun(runId);
|
|
438
|
+
if (settled?.wire?.type === 'workflow' && isUnwound(settled.status)) {
|
|
439
|
+
throw childRunFailure(settled) ?? error;
|
|
440
|
+
}
|
|
435
441
|
throw error;
|
|
436
442
|
}
|
|
437
443
|
// A lease error says nothing about how the run went — a lost lease is
|
|
@@ -455,7 +461,9 @@ export class PikkuWorkflowService {
|
|
|
455
461
|
async runToCompletion(name, input, rpcService, options) {
|
|
456
462
|
const { runId } = await this.startWorkflow(name, input, options?.wire ?? { type: 'internal' }, rpcService, { inline: true });
|
|
457
463
|
const run = await this.awaitRunEnd(runId, options?.pollIntervalMs ?? 1000);
|
|
458
|
-
if (run.status === 'failed'
|
|
464
|
+
if (run.status === 'failed' ||
|
|
465
|
+
run.status === 'compensated' ||
|
|
466
|
+
run.status === 'compensation_failed') {
|
|
459
467
|
throw new WorkflowRunFailedError(run.error?.message);
|
|
460
468
|
}
|
|
461
469
|
if (run.status === 'cancelled') {
|
|
@@ -565,6 +573,11 @@ export class PikkuWorkflowService {
|
|
|
565
573
|
}
|
|
566
574
|
if (isRunSettled(run.status))
|
|
567
575
|
return;
|
|
576
|
+
if (run.status === 'compensating') {
|
|
577
|
+
const host = createUnwindHost(this.compensationApi(rpcService));
|
|
578
|
+
await this.withRunLease(runId, () => driveUnwind(host, runId));
|
|
579
|
+
return;
|
|
580
|
+
}
|
|
568
581
|
const resolved = resolveWorkflowMeta(run.workflow);
|
|
569
582
|
const workflowMeta = resolved?.meta;
|
|
570
583
|
const pkgName = resolved?.packageName ?? null;
|
|
@@ -579,13 +592,12 @@ export class PikkuWorkflowService {
|
|
|
579
592
|
// can fire the same next node; the run lease keeps them one at a time,
|
|
580
593
|
// exactly as it does for a DSL run's replay.
|
|
581
594
|
await this.withRunLease(runId, async () => {
|
|
582
|
-
await continueGraph(this, runId, run.workflow);
|
|
595
|
+
await continueGraph(this, runId, run.workflow, undefined, rpcService);
|
|
583
596
|
const updatedRun = await this.getRun(runId);
|
|
584
597
|
if (updatedRun?.status === 'completed') {
|
|
585
598
|
await this.onChildWorkflowCompleted(updatedRun, updatedRun.output);
|
|
586
599
|
}
|
|
587
|
-
else if (updatedRun?.status === '
|
|
588
|
-
updatedRun?.status === 'cancelled') {
|
|
600
|
+
else if (updatedRun?.status === 'cancelled') {
|
|
589
601
|
await this.onChildWorkflowFailed(updatedRun, new Error(updatedRun.error?.message || 'Child workflow failed'));
|
|
590
602
|
}
|
|
591
603
|
});
|
|
@@ -599,82 +611,14 @@ export class PikkuWorkflowService {
|
|
|
599
611
|
if (!workflow) {
|
|
600
612
|
throw new WorkflowNotFoundError(run.workflow);
|
|
601
613
|
}
|
|
602
|
-
await this.withRunLease(runId,
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
workflow: workflowWire,
|
|
611
|
-
pikkuUserId: run.wire?.pikkuUserId,
|
|
612
|
-
};
|
|
613
|
-
this.runExtension?.decorateRunWire(wire, {
|
|
614
|
-
runId,
|
|
615
|
-
workflowMeta,
|
|
616
|
-
workflowWire,
|
|
617
|
-
});
|
|
618
|
-
const lifecycle = {
|
|
619
|
-
runId,
|
|
620
|
-
run,
|
|
621
|
-
workflowMeta,
|
|
622
|
-
workflow,
|
|
623
|
-
wire,
|
|
624
|
-
packageName: pkgName,
|
|
625
|
-
};
|
|
626
|
-
let outcome = 'completed';
|
|
627
|
-
let failure;
|
|
628
|
-
try {
|
|
629
|
-
await this.runExtension?.onBeforeRunFunc(lifecycle);
|
|
630
|
-
const result = await runPikkuFunc('workflow', workflowMeta.name, workflowMeta.pikkuFuncId, {
|
|
631
|
-
singletonServices: getSingletonServices(),
|
|
632
|
-
wire,
|
|
633
|
-
createWireServices: getCreateWireServices(),
|
|
634
|
-
data: () => run.input,
|
|
635
|
-
packageName: pkgName ?? undefined,
|
|
636
|
-
});
|
|
637
|
-
await this.updateRunStatus(runId, 'completed', result);
|
|
638
|
-
await this.onChildWorkflowCompleted(run, result);
|
|
639
|
-
}
|
|
640
|
-
catch (error) {
|
|
641
|
-
failure = error;
|
|
642
|
-
if (error instanceof WorkflowAsyncException) {
|
|
643
|
-
outcome = 'interrupted';
|
|
644
|
-
throw error;
|
|
645
|
-
}
|
|
646
|
-
if (error instanceof WorkflowCancelledException) {
|
|
647
|
-
outcome = 'failed';
|
|
648
|
-
await this.updateRunStatus(runId, 'cancelled', undefined, {
|
|
649
|
-
message: error.message || 'Workflow cancelled',
|
|
650
|
-
stack: '',
|
|
651
|
-
code: 'WORKFLOW_CANCELLED',
|
|
652
|
-
});
|
|
653
|
-
await this.onChildWorkflowFailed(run, error);
|
|
654
|
-
throw error;
|
|
655
|
-
}
|
|
656
|
-
if (error instanceof WorkflowSuspendedException) {
|
|
657
|
-
outcome = 'interrupted';
|
|
658
|
-
await this.updateRunStatus(runId, 'suspended', undefined, {
|
|
659
|
-
message: error.message || 'Workflow suspended',
|
|
660
|
-
stack: '',
|
|
661
|
-
code: 'WORKFLOW_SUSPENDED',
|
|
662
|
-
});
|
|
663
|
-
throw error;
|
|
664
|
-
}
|
|
665
|
-
outcome = 'failed';
|
|
666
|
-
await this.updateRunStatus(runId, 'failed', undefined, {
|
|
667
|
-
message: error.message,
|
|
668
|
-
stack: error.stack,
|
|
669
|
-
code: error.code,
|
|
670
|
-
});
|
|
671
|
-
await this.onChildWorkflowFailed(run, error);
|
|
672
|
-
throw error;
|
|
673
|
-
}
|
|
674
|
-
finally {
|
|
675
|
-
await this.runExtension?.onAfterRunFunc(lifecycle, outcome, failure);
|
|
676
|
-
}
|
|
677
|
-
});
|
|
614
|
+
await this.withRunLease(runId, () => runDslWorkflowPass({
|
|
615
|
+
workflowWire: (addonNs) => this.createWorkflowWire(run.workflow, runId, rpcService, addonNs),
|
|
616
|
+
extension: this.runExtension,
|
|
617
|
+
updateRunStatus: (status, output, error) => this.updateRunStatus(runId, status, output, error),
|
|
618
|
+
onChildCompleted: (result) => this.onChildWorkflowCompleted(run, result),
|
|
619
|
+
onChildFailed: (error) => this.onChildWorkflowFailed(run, error),
|
|
620
|
+
failOrUnwind: (error) => this.failOrUnwind(run, error, rpcService),
|
|
621
|
+
}, run, workflowMeta, workflow, pkgName));
|
|
678
622
|
}
|
|
679
623
|
async onChildWorkflowCompleted(childRun, result) {
|
|
680
624
|
const { parentRunId, parentStepId } = childRun.wire ?? {};
|
|
@@ -684,6 +628,9 @@ export class PikkuWorkflowService {
|
|
|
684
628
|
await this.setStepResult(parentStepId, result);
|
|
685
629
|
await this.resumeWorkflow(parentRunId);
|
|
686
630
|
}
|
|
631
|
+
notifyChildWorkflowFailed(childRun, error) {
|
|
632
|
+
return this.onChildWorkflowFailed(childRun, error);
|
|
633
|
+
}
|
|
687
634
|
async onChildWorkflowFailed(childRun, error) {
|
|
688
635
|
const { parentRunId, parentStepId } = childRun.wire ?? {};
|
|
689
636
|
if (!parentRunId || !parentStepId)
|
|
@@ -761,34 +708,20 @@ export class PikkuWorkflowService {
|
|
|
761
708
|
result = await executeGraphStep(this, rpcService, runId, stepState.stepId, graphNodeId, workflowMeta.nodes[graphNodeId].rpcName, data, run.workflow);
|
|
762
709
|
}
|
|
763
710
|
else {
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
parentStepId: stepState.stepId,
|
|
771
|
-
pikkuUserId: run.wire?.pikkuUserId,
|
|
772
|
-
};
|
|
773
|
-
const shouldInline = !getSingletonServices()?.queueService;
|
|
774
|
-
const { runId: childRunId } = await this.startWorkflow(rpcName, data, childWire, rpcService, { inline: shouldInline });
|
|
775
|
-
await this.setStepChildRunId(stepState.stepId, childRunId);
|
|
776
|
-
if (shouldInline) {
|
|
777
|
-
const childRun = await this.getRun(childRunId);
|
|
778
|
-
if (childRun?.status === 'failed') {
|
|
779
|
-
throw new Error(childRun.error?.message || 'Sub-workflow failed');
|
|
780
|
-
}
|
|
781
|
-
if (childRun?.status === 'cancelled') {
|
|
782
|
-
throw new Error('Sub-workflow was cancelled');
|
|
783
|
-
}
|
|
784
|
-
result = childRun?.output;
|
|
785
|
-
}
|
|
786
|
-
else {
|
|
787
|
-
throw new ChildWorkflowStartedException(runId, stepState.stepId, childRunId);
|
|
788
|
-
}
|
|
711
|
+
if (meta[rpcName]) {
|
|
712
|
+
result = await startQueuedChildStep({
|
|
713
|
+
startWorkflow: (...args) => this.startWorkflow(...args),
|
|
714
|
+
setStepChildRunId: (...args) => this.setStepChildRunId(...args),
|
|
715
|
+
getRun: (id) => this.getRun(id),
|
|
716
|
+
}, runId, stepState.stepId, rpcName, data, rpcService, run.wire?.pikkuUserId);
|
|
789
717
|
}
|
|
790
718
|
else {
|
|
791
|
-
result = await this.invokeStepRpc(runId, stepName, stepState, rpcName, data, rpcService, run)
|
|
719
|
+
result = await this.invokeStepRpc(runId, stepName, stepState, rpcName, data, rpcService, run, isCompensationStepName(stepName)
|
|
720
|
+
? compensatingForStep({
|
|
721
|
+
...(await this.getStepState(runId, forwardStepName(stepName))),
|
|
722
|
+
stepName: forwardStepName(stepName),
|
|
723
|
+
})
|
|
724
|
+
: undefined);
|
|
792
725
|
}
|
|
793
726
|
}
|
|
794
727
|
await this.setStepResult(stepState.stepId, result, attempt);
|
|
@@ -848,7 +781,7 @@ export class PikkuWorkflowService {
|
|
|
848
781
|
getSingletonServices().logger.warn(`Workflow run ${runId} could not dispatch a step (queue unavailable); leaving run for orchestrator retry`, error);
|
|
849
782
|
throw error;
|
|
850
783
|
}
|
|
851
|
-
await this.
|
|
784
|
+
await this.markFailedUnlessUnwound(runId, {
|
|
852
785
|
message: error.message,
|
|
853
786
|
stack: error.stack,
|
|
854
787
|
code: error.code,
|
|
@@ -856,10 +789,18 @@ export class PikkuWorkflowService {
|
|
|
856
789
|
throw error;
|
|
857
790
|
}
|
|
858
791
|
}
|
|
859
|
-
async invokeStepRpc(runId, stepName, stepState, rpcName, data, rpcService, knownRun) {
|
|
792
|
+
async invokeStepRpc(runId, stepName, stepState, rpcName, data, rpcService, knownRun, compensatingFor) {
|
|
860
793
|
const run = knownRun ?? (await this.getRunIdentity(runId));
|
|
861
794
|
return rpcService.rpcWithWire(rpcName, data, {
|
|
862
795
|
...(run?.wire?.pikkuUserId ? { pikkuUserId: run.wire.pikkuUserId } : {}),
|
|
796
|
+
...(compensatingFor
|
|
797
|
+
? {
|
|
798
|
+
workflow: {
|
|
799
|
+
...this.createWorkflowWire(run?.workflow ?? '', runId, rpcService),
|
|
800
|
+
compensatingFor,
|
|
801
|
+
},
|
|
802
|
+
}
|
|
803
|
+
: {}),
|
|
863
804
|
workflowStep: {
|
|
864
805
|
runId,
|
|
865
806
|
stepId: stepState.stepId,
|
|
@@ -878,18 +819,19 @@ export class PikkuWorkflowService {
|
|
|
878
819
|
retries: stepOptions?.retries ?? DEFAULT_STEP_RETRIES,
|
|
879
820
|
retryDelay: stepOptions?.retryDelay,
|
|
880
821
|
actor: stepOptions?.actor,
|
|
881
|
-
onError: stepOptions?.onError,
|
|
882
822
|
};
|
|
883
|
-
const stepState = await this.loadOrCreateStep(runId, stepName, () =>
|
|
823
|
+
const stepState = await this.loadOrCreateStep(runId, stepName, async () => {
|
|
824
|
+
if (stepOptions?.compensate === false) {
|
|
825
|
+
await this.updateRunState(runId, `${NO_COMPENSATE_STATE_PREFIX}${stepName}`, true);
|
|
826
|
+
}
|
|
827
|
+
return this.insertStepState(runId, stepName, rpcName, data, resolvedStepOptions, fromStepName);
|
|
828
|
+
});
|
|
884
829
|
if (stepState.status === 'succeeded') {
|
|
885
830
|
return stepState.result;
|
|
886
831
|
}
|
|
887
832
|
if (stepState.status === 'failed') {
|
|
888
833
|
const error = new Error(stepState.error?.message ||
|
|
889
834
|
`Step '${stepName}' failed after exhausting all retries`);
|
|
890
|
-
if (resolvedStepOptions.onError) {
|
|
891
|
-
await this.rpcStep(runId, `${stepName}:onError`, resolvedStepOptions.onError, { error: { message: error.message } }, rpcService, { retries: 0 });
|
|
892
|
-
}
|
|
893
835
|
if (stepState.error) {
|
|
894
836
|
Object.assign(error, stepState.error);
|
|
895
837
|
}
|
|
@@ -917,24 +859,12 @@ export class PikkuWorkflowService {
|
|
|
917
859
|
if (resolvedStepOptions.actor) {
|
|
918
860
|
return resolvedStepOptions.actor.invoke(rpcName, data);
|
|
919
861
|
}
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
pikkuUserId: (await this.getRunIdentity(runId))?.wire?.pikkuUserId,
|
|
927
|
-
};
|
|
928
|
-
const { runId: childRunId } = await this.startWorkflow(rpcName, data, childWire, rpcService, { inline: true });
|
|
929
|
-
await this.setStepChildRunId(currentStepState.stepId, childRunId);
|
|
930
|
-
const childRun = await this.awaitRunEnd(childRunId, WORKFLOW_CHILD_POLL_MAX_MS);
|
|
931
|
-
if (childRun.status === 'failed') {
|
|
932
|
-
throw new Error(childRun.error?.message || 'Sub-workflow failed');
|
|
933
|
-
}
|
|
934
|
-
if (childRun.status === 'cancelled') {
|
|
935
|
-
throw new Error('Sub-workflow was cancelled');
|
|
936
|
-
}
|
|
937
|
-
return childRun.output;
|
|
862
|
+
if (pikkuState(null, 'workflows', 'meta')[rpcName]) {
|
|
863
|
+
return runInlineChildStep({
|
|
864
|
+
startWorkflow: (...args) => this.startWorkflow(...args),
|
|
865
|
+
setStepChildRunId: (...args) => this.setStepChildRunId(...args),
|
|
866
|
+
awaitRunEnd: (id) => this.awaitRunEnd(id, WORKFLOW_CHILD_POLL_MAX_MS),
|
|
867
|
+
}, runId, currentStepState.stepId, rpcName, data, rpcService, (await this.getRunIdentity(runId))?.wire?.pikkuUserId);
|
|
938
868
|
}
|
|
939
869
|
return this.invokeStepRpc(runId, stepName, currentStepState, rpcName, data, rpcService);
|
|
940
870
|
}, async (error) => {
|
|
@@ -1020,14 +950,14 @@ export class PikkuWorkflowService {
|
|
|
1020
950
|
}
|
|
1021
951
|
get approvalStore() {
|
|
1022
952
|
return {
|
|
1023
|
-
getStepState: (
|
|
1024
|
-
insertStepState: (
|
|
1025
|
-
setStepRunning: (
|
|
1026
|
-
setStepResult: (
|
|
1027
|
-
getRunState: (
|
|
1028
|
-
updateRunState: (
|
|
953
|
+
getStepState: (...args) => this.getStepState(...args),
|
|
954
|
+
insertStepState: (...args) => this.insertStepState(...args),
|
|
955
|
+
setStepRunning: (...args) => this.setStepRunning(...args),
|
|
956
|
+
setStepResult: (...args) => this.setStepResult(...args),
|
|
957
|
+
getRunState: (...args) => this.getRunState(...args),
|
|
958
|
+
updateRunState: (...args) => this.updateRunState(...args),
|
|
1029
959
|
resumeWorkflow: (runId) => this.resumeWorkflow(runId),
|
|
1030
|
-
scheduleRunWake: (
|
|
960
|
+
scheduleRunWake: (...args) => this.scheduleRunWake(...args),
|
|
1031
961
|
getRunOwner: async (runId) => (await this.getRunIdentity(runId))?.wire?.pikkuUserId,
|
|
1032
962
|
auditApproval: (event) => auditApprovalDecision(event),
|
|
1033
963
|
};
|
|
@@ -1065,6 +995,12 @@ export class PikkuWorkflowService {
|
|
|
1065
995
|
this.verifyStepName(reason);
|
|
1066
996
|
await this.suspendStep(runId, reason);
|
|
1067
997
|
},
|
|
998
|
+
milestone: async (name) => {
|
|
999
|
+
this.verifyStepName(name);
|
|
1000
|
+
await this.inlineStep(runId, milestoneStepName(name), async () => ({
|
|
1001
|
+
name,
|
|
1002
|
+
}));
|
|
1003
|
+
},
|
|
1068
1004
|
approval: (async (reason, options) => {
|
|
1069
1005
|
this.verifyStepName(reason);
|
|
1070
1006
|
return await this.approvalStep(runId, reason, options);
|
|
@@ -1078,6 +1014,40 @@ export class PikkuWorkflowService {
|
|
|
1078
1014
|
});
|
|
1079
1015
|
return workflowWire;
|
|
1080
1016
|
}
|
|
1017
|
+
compensationApi(rpcService) {
|
|
1018
|
+
return {
|
|
1019
|
+
getRun: (...args) => this.getRun(...args),
|
|
1020
|
+
getRunSteps: (...args) => this.getRunSteps(...args),
|
|
1021
|
+
getRunState: (...args) => this.getRunState(...args),
|
|
1022
|
+
updateRunStatus: (...args) => this.updateRunStatus(...args),
|
|
1023
|
+
loadStep: (...args) => this.loadOrCreateStep(...args),
|
|
1024
|
+
insertStepState: (...args) => this.insertStepState(...args),
|
|
1025
|
+
dispatchStep: (...args) => this.dispatchStep(...args),
|
|
1026
|
+
setStepScheduled: (id) => this.setStepScheduled(id),
|
|
1027
|
+
setStepResult: (id, result) => this.setStepResult(id, result),
|
|
1028
|
+
setStepError: (id, error) => this.setStepError(id, error),
|
|
1029
|
+
runInline: (...args) => runInlineRetryLoop(this, ...args),
|
|
1030
|
+
invoke: (runId, stepName, step, rpcName, data, compensatingFor) => this.invokeStepRpc(runId, stepName, step, rpcName, data, rpcService, undefined, compensatingFor),
|
|
1031
|
+
failOrUnwind: (run, error) => this.failOrUnwind(run, error, rpcService),
|
|
1032
|
+
notifyParent: (run, error) => this.onChildWorkflowFailed(run, error),
|
|
1033
|
+
hasCompensation: recordHasCompensation,
|
|
1034
|
+
isChildWorkflow: isChildWorkflowRecord,
|
|
1035
|
+
};
|
|
1036
|
+
}
|
|
1037
|
+
async failOrUnwind(run, error, rpcService) {
|
|
1038
|
+
const api = this.compensationApi(rpcService);
|
|
1039
|
+
return failRunOrUnwind(api, run, error, () => this.onChildWorkflowFailed(run, error));
|
|
1040
|
+
}
|
|
1041
|
+
/** Stops an unfinished run, children first, and unwinds what it completed. */
|
|
1042
|
+
async cancelRun(runId, rpcService, reason) {
|
|
1043
|
+
const api = this.compensationApi(rpcService);
|
|
1044
|
+
return cancelAndUnwind(api, createUnwindHost(api), runId, reason);
|
|
1045
|
+
}
|
|
1046
|
+
async markFailedUnlessUnwound(runId, error) {
|
|
1047
|
+
if (isUnwound((await this.getRun(runId))?.status))
|
|
1048
|
+
return;
|
|
1049
|
+
await this.updateRunStatus(runId, 'failed', undefined, error);
|
|
1050
|
+
}
|
|
1081
1051
|
verifyStepName(stepName) {
|
|
1082
1052
|
if (typeof stepName !== 'string') {
|
|
1083
1053
|
throw new WorkflowStepNameNotString(stepName);
|
|
@@ -5,6 +5,7 @@ export interface RunTimelineEvent {
|
|
|
5
5
|
at: Date;
|
|
6
6
|
type: Extract<StepStatus, 'pending' | 'scheduled' | 'running' | 'succeeded' | 'failed'>;
|
|
7
7
|
stepName: string;
|
|
8
|
+
compensating: boolean;
|
|
8
9
|
attemptCount: number;
|
|
9
10
|
fromStepName?: string;
|
|
10
11
|
result?: unknown;
|
|
@@ -17,6 +18,7 @@ export type HistoryEntry = StepState & {
|
|
|
17
18
|
export declare function buildRunTimeline(history: HistoryEntry[]): RunTimeline;
|
|
18
19
|
export interface ReconstructedStep {
|
|
19
20
|
stepName: string;
|
|
21
|
+
compensating: boolean;
|
|
20
22
|
status: StepStatus;
|
|
21
23
|
attemptCount: number;
|
|
22
24
|
fromStepName?: string;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { isCompensationStepName } from './workflow-constants.js';
|
|
1
2
|
const LIFECYCLE_ORDER = {
|
|
2
3
|
pending: 0,
|
|
3
4
|
scheduled: 1,
|
|
@@ -10,6 +11,7 @@ export function buildRunTimeline(history) {
|
|
|
10
11
|
history.forEach((entry, order) => {
|
|
11
12
|
const base = {
|
|
12
13
|
stepName: entry.stepName,
|
|
14
|
+
compensating: entry.compensating ?? isCompensationStepName(entry.stepName),
|
|
13
15
|
attemptCount: entry.attemptCount,
|
|
14
16
|
order,
|
|
15
17
|
};
|
|
@@ -77,6 +79,7 @@ export function reconstructStateAt(timeline, at) {
|
|
|
77
79
|
if (!step) {
|
|
78
80
|
step = {
|
|
79
81
|
stepName: event.stepName,
|
|
82
|
+
compensating: event.compensating,
|
|
80
83
|
status: event.type,
|
|
81
84
|
attemptCount: event.attemptCount,
|
|
82
85
|
fromStepName: event.fromStepName,
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { PikkuRPC } from '../rpc/rpc-types.js';
|
|
2
|
+
import type { WorkflowRun, WorkflowRunWire } from './workflow.types.js';
|
|
3
|
+
interface ChildStepApi {
|
|
4
|
+
startWorkflow(name: string, input: unknown, wire: WorkflowRunWire, rpcService: PikkuRPC, options?: {
|
|
5
|
+
inline?: boolean;
|
|
6
|
+
}): Promise<{
|
|
7
|
+
runId: string;
|
|
8
|
+
}>;
|
|
9
|
+
setStepChildRunId(stepId: string, childRunId: string): Promise<void>;
|
|
10
|
+
}
|
|
11
|
+
export declare const runInlineChildStep: (api: ChildStepApi & {
|
|
12
|
+
awaitRunEnd(childRunId: string): Promise<WorkflowRun>;
|
|
13
|
+
}, runId: string, stepId: string, workflowName: string, data: unknown, rpcService: PikkuRPC, pikkuUserId: string | undefined) => Promise<unknown>;
|
|
14
|
+
export declare const startQueuedChildStep: (api: ChildStepApi & {
|
|
15
|
+
getRun(id: string): Promise<WorkflowRun | null>;
|
|
16
|
+
}, runId: string, stepId: string, workflowName: string, data: unknown, rpcService: PikkuRPC, pikkuUserId: string | undefined) => Promise<unknown>;
|
|
17
|
+
export {};
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { getSingletonServices } from '../../pikku-state.js';
|
|
2
|
+
import { ChildWorkflowStartedException } from './graph/graph-runner.js';
|
|
3
|
+
import { childRunFailure } from './workflow-compensation.js';
|
|
4
|
+
export const runInlineChildStep = async (api, runId, stepId, workflowName, data, rpcService, pikkuUserId) => {
|
|
5
|
+
const { runId: childRunId } = await api.startWorkflow(workflowName, data, { type: 'workflow', id: workflowName, parentRunId: runId, pikkuUserId }, rpcService, { inline: true });
|
|
6
|
+
await api.setStepChildRunId(stepId, childRunId);
|
|
7
|
+
const childRun = await api.awaitRunEnd(childRunId);
|
|
8
|
+
const failure = childRunFailure(childRun);
|
|
9
|
+
if (failure)
|
|
10
|
+
throw failure;
|
|
11
|
+
return childRun.output;
|
|
12
|
+
};
|
|
13
|
+
export const startQueuedChildStep = async (api, runId, stepId, workflowName, data, rpcService, pikkuUserId) => {
|
|
14
|
+
const shouldInline = !getSingletonServices()?.queueService;
|
|
15
|
+
const { runId: childRunId } = await api.startWorkflow(workflowName, data, {
|
|
16
|
+
type: 'workflow',
|
|
17
|
+
id: workflowName,
|
|
18
|
+
parentRunId: runId,
|
|
19
|
+
parentStepId: stepId,
|
|
20
|
+
pikkuUserId,
|
|
21
|
+
}, rpcService, { inline: shouldInline });
|
|
22
|
+
await api.setStepChildRunId(stepId, childRunId);
|
|
23
|
+
if (!shouldInline) {
|
|
24
|
+
throw new ChildWorkflowStartedException(runId, stepId, childRunId);
|
|
25
|
+
}
|
|
26
|
+
const childRun = await api.getRun(childRunId);
|
|
27
|
+
const failure = childRun && childRunFailure(childRun);
|
|
28
|
+
if (failure)
|
|
29
|
+
throw failure;
|
|
30
|
+
return childRun?.output;
|
|
31
|
+
};
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import type { SerializedError } from '../../errors/serialized-error.js';
|
|
2
|
+
import type { CompensatingFor } from './dsl/workflow-dsl.types.js';
|
|
3
|
+
import { type UnwindRecord } from './workflow-unwind-plan.js';
|
|
4
|
+
import type { StepState, WorkflowRun, WorkflowStatus, WorkflowStepOptions } from './workflow.types.js';
|
|
5
|
+
export type CompensationStepOutcome = 'done' | 'waiting' | 'stuck';
|
|
6
|
+
export interface UnwindHost {
|
|
7
|
+
getRun(runId: string): Promise<WorkflowRun | null>;
|
|
8
|
+
getRunSteps(runId: string): Promise<UnwindRecord[]>;
|
|
9
|
+
getRunState(runId: string): Promise<Record<string, unknown>>;
|
|
10
|
+
hasCompensation(record: UnwindRecord): boolean;
|
|
11
|
+
isChildWorkflow(record: UnwindRecord): boolean;
|
|
12
|
+
runCompensationStep(runId: string, forward: UnwindRecord, compensatingFor: CompensatingFor): Promise<CompensationStepOutcome>;
|
|
13
|
+
finish(run: WorkflowRun, status: WorkflowStatus, output: unknown, error: SerializedError | undefined): Promise<void>;
|
|
14
|
+
}
|
|
15
|
+
export declare const canEnterCompensation: (status: string) => boolean;
|
|
16
|
+
export declare const compensatingForStep: (forward: Pick<UnwindRecord, 'status' | 'result' | 'error' | 'stepName'>) => CompensatingFor;
|
|
17
|
+
export declare const skippedSteps: (state: Record<string, unknown>) => Set<string>;
|
|
18
|
+
export declare const wouldCompensate: (host: UnwindHost, runId: string) => Promise<boolean>;
|
|
19
|
+
export declare const driveUnwind: (host: UnwindHost, runId: string) => Promise<'waiting' | 'finished'>;
|
|
20
|
+
export declare const compensationRpcName: (rpcName: string) => string;
|
|
21
|
+
export interface CompensationApi {
|
|
22
|
+
getRun(runId: string): Promise<WorkflowRun | null>;
|
|
23
|
+
getRunSteps(runId: string): Promise<UnwindRecord[]>;
|
|
24
|
+
getRunState(runId: string): Promise<Record<string, unknown>>;
|
|
25
|
+
updateRunStatus(runId: string, status: WorkflowStatus, output?: unknown, error?: SerializedError): Promise<void>;
|
|
26
|
+
loadStep(runId: string, stepName: string, create: () => Promise<StepState>): Promise<StepState>;
|
|
27
|
+
insertStepState(runId: string, stepName: string, rpcName: string | null, data: unknown, options: WorkflowStepOptions, fromStepName: string): Promise<StepState>;
|
|
28
|
+
dispatchStep(runId: string, stepName: string, rpcName: string, data: unknown, options: WorkflowStepOptions, fromStepName: string): Promise<boolean>;
|
|
29
|
+
setStepScheduled(stepId: string): Promise<void>;
|
|
30
|
+
runInline(stepState: StepState, retries: number, retryDelay: WorkflowStepOptions['retryDelay'], work: (current: StepState) => Promise<unknown>): Promise<unknown>;
|
|
31
|
+
invoke(runId: string, stepName: string, stepState: StepState, rpcName: string, data: unknown, compensatingFor: CompensatingFor): Promise<unknown>;
|
|
32
|
+
failOrUnwind(run: WorkflowRun, error: Error): Promise<unknown>;
|
|
33
|
+
setStepResult(stepId: string, result: unknown): Promise<void>;
|
|
34
|
+
setStepError(stepId: string, error: Error): Promise<void>;
|
|
35
|
+
notifyParent(run: WorkflowRun, error: Error): Promise<void>;
|
|
36
|
+
hasCompensation(record: UnwindRecord): boolean;
|
|
37
|
+
isChildWorkflow(record: UnwindRecord): boolean;
|
|
38
|
+
}
|
|
39
|
+
export declare const createUnwindHost: (api: CompensationApi) => UnwindHost;
|
|
40
|
+
export declare const childCompensationError: (status: WorkflowStatus, output: unknown, error: SerializedError | undefined) => Error & {
|
|
41
|
+
childCompensation: ChildCompensationInfo;
|
|
42
|
+
};
|
|
43
|
+
export declare const functionHasCompensation: (rpcName: string) => boolean;
|
|
44
|
+
export declare const isChildWorkflowRecord: (record: UnwindRecord) => boolean;
|
|
45
|
+
export declare const recordHasCompensation: (record: UnwindRecord) => boolean;
|
|
46
|
+
export interface ChildCompensationInfo {
|
|
47
|
+
status: WorkflowStatus;
|
|
48
|
+
restedAt?: string;
|
|
49
|
+
stuckSteps?: Array<{
|
|
50
|
+
stepName: string;
|
|
51
|
+
error: string;
|
|
52
|
+
}>;
|
|
53
|
+
}
|
|
54
|
+
export type UnwindCause = SerializedError & {
|
|
55
|
+
childCompensation?: ChildCompensationInfo;
|
|
56
|
+
};
|
|
57
|
+
export declare const childRunFailure: (childRun: WorkflowRun) => Error | null;
|
|
58
|
+
export declare const beginUnwind: (api: CompensationApi, host: UnwindHost, run: WorkflowRun, cause: UnwindCause) => Promise<'unwinding' | 'finished' | 'not-needed'>;
|
|
59
|
+
export declare const isUnwound: (status: string | undefined) => boolean;
|
|
60
|
+
export declare const failRunOrUnwind: (api: CompensationApi, run: WorkflowRun, error: Error & {
|
|
61
|
+
code?: string;
|
|
62
|
+
childCompensation?: ChildCompensationInfo;
|
|
63
|
+
}, onFailed: () => Promise<void>) => Promise<'unwinding' | 'finished' | 'failed'>;
|
|
64
|
+
/**
|
|
65
|
+
* Child workflows are cancelled first so nothing keeps producing effects under
|
|
66
|
+
* a parent that is unwinding. A run with nothing to undo stays `cancelled`.
|
|
67
|
+
*/
|
|
68
|
+
export declare const cancelAndUnwind: (api: CompensationApi, host: UnwindHost, runId: string, reason?: string) => Promise<WorkflowStatus | undefined>;
|