bunqueue 2.8.49 → 2.8.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -9
- package/dist/application/contextFactory.d.ts +4 -2
- package/dist/application/contextFactory.js +3 -0
- package/dist/application/flowFailureRecovery.d.ts +17 -0
- package/dist/application/flowFailureRecovery.js +92 -0
- package/dist/application/operations/ack.d.ts +3 -2
- package/dist/application/operations/ack.js +34 -8
- package/dist/application/operations/flowPush.d.ts +9 -0
- package/dist/application/operations/flowPush.js +112 -0
- package/dist/application/operations/flowTopologyValidation.d.ts +3 -0
- package/dist/application/operations/flowTopologyValidation.js +101 -0
- package/dist/application/operations/flowValidation.d.ts +3 -0
- package/dist/application/operations/flowValidation.js +166 -0
- package/dist/application/operations/push.d.ts +3 -0
- package/dist/application/operations/push.js +2 -2
- package/dist/application/operations/pushInsert.d.ts +7 -1
- package/dist/application/operations/pushInsert.js +18 -8
- package/dist/application/operations/pushLocks.d.ts +1 -0
- package/dist/application/operations/pushLocks.js +5 -1
- package/dist/application/operations/queryOperations.js +7 -6
- package/dist/application/queueManager.d.ts +5 -0
- package/dist/application/queueManager.js +195 -60
- package/dist/cli/commandRouter.d.ts +1 -1
- package/dist/client/flow.d.ts +3 -6
- package/dist/client/flow.js +71 -210
- package/dist/client/flowAtomic.d.ts +9 -0
- package/dist/client/flowAtomic.js +24 -0
- package/dist/client/flowJobCoreMethods.d.ts +28 -0
- package/dist/client/flowJobCoreMethods.js +151 -0
- package/dist/client/flowJobDependencies.d.ts +7 -0
- package/dist/client/flowJobDependencies.js +64 -0
- package/dist/client/flowJobFactory.d.ts +11 -24
- package/dist/client/flowJobFactory.js +91 -360
- package/dist/client/flowJobMoveMethods.d.ts +16 -0
- package/dist/client/flowJobMoveMethods.js +112 -0
- package/dist/client/flowJobTypes.d.ts +29 -0
- package/dist/client/flowJobTypes.js +6 -0
- package/dist/client/flowLegacyPlan.d.ts +16 -0
- package/dist/client/flowLegacyPlan.js +118 -0
- package/dist/client/flowOptions.d.ts +10 -0
- package/dist/client/flowOptions.js +44 -0
- package/dist/client/flowPlan.d.ts +16 -0
- package/dist/client/flowPlan.js +93 -0
- package/dist/client/flowPush.js +4 -1
- package/dist/client/flowReader.d.ts +10 -0
- package/dist/client/flowReader.js +147 -0
- package/dist/client/jobHelpers.js +2 -0
- package/dist/client/workflow/clock.d.ts +3 -1
- package/dist/client/workflow/clock.js +18 -1
- package/dist/client/workflow/compensationChild.d.ts +30 -0
- package/dist/client/workflow/compensationChild.js +65 -0
- package/dist/client/workflow/compensationPass.d.ts +16 -0
- package/dist/client/workflow/compensationPass.js +96 -0
- package/dist/client/workflow/compensationSupport.d.ts +24 -0
- package/dist/client/workflow/compensationSupport.js +57 -0
- package/dist/client/workflow/compensator.d.ts +11 -50
- package/dist/client/workflow/compensator.js +22 -374
- package/dist/client/workflow/definitionGuard.d.ts +11 -0
- package/dist/client/workflow/definitionGuard.js +22 -0
- package/dist/client/workflow/engine.d.ts +2 -2
- package/dist/client/workflow/engine.js +8 -8
- package/dist/client/workflow/eventTypes.d.ts +44 -0
- package/dist/client/workflow/eventTypes.js +2 -0
- package/dist/client/workflow/executionTypes.d.ts +133 -0
- package/dist/client/workflow/executionTypes.js +2 -0
- package/dist/client/workflow/executor.d.ts +4 -17
- package/dist/client/workflow/executor.js +50 -208
- package/dist/client/workflow/executorLifecycle.d.ts +18 -0
- package/dist/client/workflow/executorLifecycle.js +69 -0
- package/dist/client/workflow/executorNodes.d.ts +17 -0
- package/dist/client/workflow/executorNodes.js +138 -0
- package/dist/client/workflow/identity.d.ts +2 -0
- package/dist/client/workflow/identity.js +8 -0
- package/dist/client/workflow/index.d.ts +1 -1
- package/dist/client/workflow/loops.d.ts +3 -5
- package/dist/client/workflow/loops.js +25 -23
- package/dist/client/workflow/mapRunner.d.ts +4 -0
- package/dist/client/workflow/mapRunner.js +45 -0
- package/dist/client/workflow/recovery.js +10 -3
- package/dist/client/workflow/runner.d.ts +2 -21
- package/dist/client/workflow/runner.js +39 -104
- package/dist/client/workflow/runnerTiming.d.ts +14 -0
- package/dist/client/workflow/runnerTiming.js +66 -0
- package/dist/client/workflow/stepTypes.d.ts +171 -0
- package/dist/client/workflow/stepTypes.js +3 -0
- package/dist/client/workflow/store.d.ts +17 -17
- package/dist/client/workflow/store.js +65 -100
- package/dist/client/workflow/storeExecutionCodec.d.ts +11 -0
- package/dist/client/workflow/storeExecutionCodec.js +34 -0
- package/dist/client/workflow/storeListing.d.ts +11 -0
- package/dist/client/workflow/storeListing.js +45 -0
- package/dist/client/workflow/storeMaintenance.d.ts +4 -0
- package/dist/client/workflow/storeMaintenance.js +40 -0
- package/dist/client/workflow/storeSignals.d.ts +9 -0
- package/dist/client/workflow/storeSignals.js +38 -2
- package/dist/client/workflow/subWorkflowRunner.d.ts +13 -0
- package/dist/client/workflow/subWorkflowRunner.js +40 -0
- package/dist/client/workflow/types.d.ts +4 -356
- package/dist/client/workflow/types.js +1 -3
- package/dist/client/workflow/waitFor.js +39 -26
- package/dist/client/workflow/workflow.d.ts +16 -59
- package/dist/client/workflow/workflow.js +53 -179
- package/dist/client/workflow/workflowDecisions.d.ts +11 -0
- package/dist/client/workflow/workflowDecisions.js +27 -0
- package/dist/client/workflow/workflowDefinition.d.ts +16 -0
- package/dist/client/workflow/workflowDefinition.js +123 -0
- package/dist/client/workflow/workflowIntrospection.d.ts +5 -0
- package/dist/client/workflow/workflowIntrospection.js +46 -0
- package/dist/client/workflow/workflowValidation.d.ts +44 -0
- package/dist/client/workflow/workflowValidation.js +143 -0
- package/dist/domain/types/command.d.ts +7 -1
- package/dist/domain/types/flow.d.ts +25 -0
- package/dist/domain/types/flow.js +1 -0
- package/dist/infrastructure/persistence/schema.d.ts +2 -2
- package/dist/infrastructure/persistence/schema.js +52 -3
- package/dist/infrastructure/persistence/sqlite.d.ts +16 -0
- package/dist/infrastructure/persistence/sqlite.js +129 -7
- package/dist/infrastructure/persistence/sqliteBatch.js +10 -4
- package/dist/infrastructure/persistence/sqliteSerializer.d.ts +2 -0
- package/dist/infrastructure/persistence/sqliteSerializer.js +18 -5
- package/dist/infrastructure/persistence/statements.d.ts +4 -0
- package/dist/infrastructure/persistence/statements.js +8 -2
- package/dist/infrastructure/server/handlerRoutes.js +3 -0
- package/dist/infrastructure/server/handlers/advanced.js +2 -2
- package/dist/infrastructure/server/handlers/flow.d.ts +7 -0
- package/dist/infrastructure/server/handlers/flow.js +11 -0
- package/dist/infrastructure/server/handlers/index.d.ts +1 -0
- package/dist/infrastructure/server/handlers/index.js +1 -0
- package/package.json +4 -2
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { type JobId } from '../domain/types/job';
|
|
2
|
+
import type { AtomicFlowBatchInput } from '../domain/types/flow';
|
|
3
|
+
import type { FlowStep } from './flowTypes';
|
|
4
|
+
export declare function planChain<T>(steps: FlowStep<T>[]): {
|
|
5
|
+
batch: AtomicFlowBatchInput;
|
|
6
|
+
ids: JobId[];
|
|
7
|
+
};
|
|
8
|
+
export declare function planBulkThen<T>(parallel: FlowStep<T>[], final: FlowStep<T>): {
|
|
9
|
+
batch: AtomicFlowBatchInput;
|
|
10
|
+
parallelIds: JobId[];
|
|
11
|
+
finalId: JobId;
|
|
12
|
+
};
|
|
13
|
+
export declare function planTree<T>(root: FlowStep<T>): {
|
|
14
|
+
batch: AtomicFlowBatchInput;
|
|
15
|
+
ids: JobId[];
|
|
16
|
+
};
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import { generateJobId, jobId } from '../domain/types/job';
|
|
2
|
+
import { flowJobInput } from './flowOptions';
|
|
3
|
+
const MAX_FLOW_DEPTH = 100;
|
|
4
|
+
const MAX_FLOW_JOBS = 10_000;
|
|
5
|
+
function validateStep(step, depth = 0) {
|
|
6
|
+
if (!step || typeof step !== 'object')
|
|
7
|
+
throw new Error('flow step must be an object');
|
|
8
|
+
if (!step.name || typeof step.name !== 'string' || step.name.length > 256) {
|
|
9
|
+
throw new Error('flow job name must be a non-empty string of at most 256 characters');
|
|
10
|
+
}
|
|
11
|
+
if (!step.queueName || typeof step.queueName !== 'string') {
|
|
12
|
+
throw new Error('flow queueName is required');
|
|
13
|
+
}
|
|
14
|
+
if (depth > MAX_FLOW_DEPTH) {
|
|
15
|
+
throw new Error(`flow exceeds the ${MAX_FLOW_DEPTH} level depth limit`);
|
|
16
|
+
}
|
|
17
|
+
if (step.children !== undefined && !Array.isArray(step.children)) {
|
|
18
|
+
throw new Error('flow children must be an array');
|
|
19
|
+
}
|
|
20
|
+
if (typeof step.data !== 'object' || step.data === null || Array.isArray(step.data)) {
|
|
21
|
+
throw new Error('flow job data must be an object');
|
|
22
|
+
}
|
|
23
|
+
for (const key of Object.keys(step.data)) {
|
|
24
|
+
if (key === 'name' || key.startsWith('__')) {
|
|
25
|
+
throw new Error(`flow job data key is reserved: ${key}`);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
function validateFlatSteps(steps) {
|
|
30
|
+
if (steps.length > MAX_FLOW_JOBS)
|
|
31
|
+
throw new Error(`flow exceeds the ${MAX_FLOW_JOBS} job limit`);
|
|
32
|
+
for (const step of steps) {
|
|
33
|
+
validateStep(step);
|
|
34
|
+
if ((step.children?.length ?? 0) > 0) {
|
|
35
|
+
throw new Error('nested children are only supported by addTree');
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
function allocate(step, ids) {
|
|
40
|
+
const raw = step.opts?.jobId;
|
|
41
|
+
if (raw !== undefined && (!raw || raw.includes(':'))) {
|
|
42
|
+
throw new Error('flow jobId must be non-empty and cannot contain a colon');
|
|
43
|
+
}
|
|
44
|
+
const id = raw === undefined ? generateJobId() : jobId(raw);
|
|
45
|
+
if (ids.has(String(id)))
|
|
46
|
+
throw new Error(`duplicate flow job id: ${String(id)}`);
|
|
47
|
+
ids.add(String(id));
|
|
48
|
+
return id;
|
|
49
|
+
}
|
|
50
|
+
function dataFor(step, internal) {
|
|
51
|
+
return { ...step.data, name: step.name, ...internal };
|
|
52
|
+
}
|
|
53
|
+
export function planChain(steps) {
|
|
54
|
+
validateFlatSteps(steps);
|
|
55
|
+
const seen = new Set();
|
|
56
|
+
const ids = steps.map((step) => allocate(step, seen));
|
|
57
|
+
const jobs = steps.map((step, index) => {
|
|
58
|
+
const dependency = index > 0 ? ids[index - 1] : undefined;
|
|
59
|
+
return {
|
|
60
|
+
id: ids[index],
|
|
61
|
+
queue: step.queueName,
|
|
62
|
+
input: flowJobInput(dataFor(step, { __flowParentId: dependency ? String(dependency) : null }), step.opts ?? {}, { dependsOn: dependency ? [dependency] : undefined }),
|
|
63
|
+
};
|
|
64
|
+
});
|
|
65
|
+
return { batch: { jobs }, ids };
|
|
66
|
+
}
|
|
67
|
+
export function planBulkThen(parallel, final) {
|
|
68
|
+
validateFlatSteps([...parallel, final]);
|
|
69
|
+
const seen = new Set();
|
|
70
|
+
const parallelIds = parallel.map((step) => allocate(step, seen));
|
|
71
|
+
const finalId = allocate(final, seen);
|
|
72
|
+
const parallelJobs = parallel.map((step, index) => ({
|
|
73
|
+
id: parallelIds[index],
|
|
74
|
+
queue: step.queueName,
|
|
75
|
+
input: flowJobInput(dataFor(step, {
|
|
76
|
+
__parentId: String(finalId),
|
|
77
|
+
__parentQueue: final.queueName,
|
|
78
|
+
}), step.opts ?? {}, { parentId: finalId }),
|
|
79
|
+
}));
|
|
80
|
+
const finalJob = {
|
|
81
|
+
id: finalId,
|
|
82
|
+
queue: final.queueName,
|
|
83
|
+
input: flowJobInput(dataFor(final, {
|
|
84
|
+
__flowParentIds: parallelIds.map(String),
|
|
85
|
+
__childrenIds: parallelIds.map(String),
|
|
86
|
+
}), final.opts ?? {}, {
|
|
87
|
+
dependsOn: parallelIds.length > 0 ? parallelIds : undefined,
|
|
88
|
+
childrenIds: parallelIds.length > 0 ? parallelIds : undefined,
|
|
89
|
+
}),
|
|
90
|
+
};
|
|
91
|
+
return { batch: { jobs: [...parallelJobs, finalJob] }, parallelIds, finalId };
|
|
92
|
+
}
|
|
93
|
+
export function planTree(root) {
|
|
94
|
+
const jobs = [];
|
|
95
|
+
const ids = [];
|
|
96
|
+
const allocated = new Set();
|
|
97
|
+
const seen = new WeakSet();
|
|
98
|
+
const visit = (step, dependency, depth = 0) => {
|
|
99
|
+
validateStep(step, depth);
|
|
100
|
+
if (seen.has(step))
|
|
101
|
+
throw new Error('flow contains a cycle or shared node');
|
|
102
|
+
seen.add(step);
|
|
103
|
+
if (jobs.length >= MAX_FLOW_JOBS) {
|
|
104
|
+
throw new Error(`flow exceeds the ${MAX_FLOW_JOBS} job limit`);
|
|
105
|
+
}
|
|
106
|
+
const id = allocate(step, allocated);
|
|
107
|
+
ids.push(id);
|
|
108
|
+
jobs.push({
|
|
109
|
+
id,
|
|
110
|
+
queue: step.queueName,
|
|
111
|
+
input: flowJobInput(dataFor(step, { __flowParentId: dependency ? String(dependency) : null }), step.opts ?? {}, { dependsOn: dependency ? [dependency] : undefined }),
|
|
112
|
+
});
|
|
113
|
+
for (const child of step.children ?? [])
|
|
114
|
+
visit(child, id, depth + 1);
|
|
115
|
+
};
|
|
116
|
+
visit(root);
|
|
117
|
+
return { batch: { jobs }, ids };
|
|
118
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { JobInput, JobId } from '../domain/types/job';
|
|
2
|
+
import type { JobOptions } from './types';
|
|
3
|
+
/** Options intentionally unsupported by BullMQ-compatible atomic flows. */
|
|
4
|
+
export declare function assertAtomicFlowOptions(opts: JobOptions): void;
|
|
5
|
+
/** Convert public JobOptions into the domain input used by the atomic broker command. */
|
|
6
|
+
export declare function flowJobInput(data: unknown, opts: JobOptions, links?: {
|
|
7
|
+
parentId?: JobId;
|
|
8
|
+
dependsOn?: JobId[];
|
|
9
|
+
childrenIds?: JobId[];
|
|
10
|
+
}): JobInput;
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
function removeFlag(value) {
|
|
2
|
+
return typeof value === 'boolean' ? value : false;
|
|
3
|
+
}
|
|
4
|
+
/** Options intentionally unsupported by BullMQ-compatible atomic flows. */
|
|
5
|
+
export function assertAtomicFlowOptions(opts) {
|
|
6
|
+
if (opts.repeat)
|
|
7
|
+
throw new Error('repeat is not supported inside an atomic flow');
|
|
8
|
+
if (opts.deduplication) {
|
|
9
|
+
throw new Error('deduplication is not supported inside an atomic flow');
|
|
10
|
+
}
|
|
11
|
+
if (opts.debounce)
|
|
12
|
+
throw new Error('debounce is not supported inside an atomic flow');
|
|
13
|
+
if (opts.parent)
|
|
14
|
+
throw new Error('FlowProducer owns parent links; opts.parent is not allowed');
|
|
15
|
+
}
|
|
16
|
+
/** Convert public JobOptions into the domain input used by the atomic broker command. */
|
|
17
|
+
export function flowJobInput(data, opts, links = {}) {
|
|
18
|
+
assertAtomicFlowOptions(opts);
|
|
19
|
+
return {
|
|
20
|
+
data,
|
|
21
|
+
priority: opts.priority,
|
|
22
|
+
delay: opts.delay,
|
|
23
|
+
maxAttempts: opts.attempts,
|
|
24
|
+
backoff: opts.backoff,
|
|
25
|
+
timeout: opts.timeout,
|
|
26
|
+
customId: opts.jobId,
|
|
27
|
+
dependsOn: links.dependsOn,
|
|
28
|
+
childrenIds: links.childrenIds,
|
|
29
|
+
parentId: links.parentId,
|
|
30
|
+
lifo: opts.lifo,
|
|
31
|
+
removeOnComplete: removeFlag(opts.removeOnComplete),
|
|
32
|
+
removeOnFail: removeFlag(opts.removeOnFail),
|
|
33
|
+
durable: opts.durable,
|
|
34
|
+
stallTimeout: opts.stallTimeout,
|
|
35
|
+
stackTraceLimit: opts.stackTraceLimit,
|
|
36
|
+
keepLogs: opts.keepLogs,
|
|
37
|
+
sizeLimit: opts.sizeLimit,
|
|
38
|
+
failParentOnFailure: opts.failParentOnFailure,
|
|
39
|
+
removeDependencyOnFailure: opts.removeDependencyOnFailure,
|
|
40
|
+
continueParentOnFailure: opts.continueParentOnFailure,
|
|
41
|
+
ignoreDependencyOnFailure: opts.ignoreDependencyOnFailure,
|
|
42
|
+
timestamp: opts.timestamp,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { type JobId } from '../domain/types/job';
|
|
2
|
+
import type { AtomicFlowBatchInput } from '../domain/types/flow';
|
|
3
|
+
import type { FlowJob, FlowOpts } from './flowTypes';
|
|
4
|
+
export interface PlannedFlowNode<T> {
|
|
5
|
+
readonly id: JobId;
|
|
6
|
+
readonly name: string;
|
|
7
|
+
readonly queueName: string;
|
|
8
|
+
readonly data: T | undefined;
|
|
9
|
+
readonly children?: PlannedFlowNode<T>[];
|
|
10
|
+
}
|
|
11
|
+
export interface FlowPlan<T> {
|
|
12
|
+
readonly batch: AtomicFlowBatchInput;
|
|
13
|
+
readonly roots: PlannedFlowNode<T>[];
|
|
14
|
+
}
|
|
15
|
+
/** Compile one or more trees into a fully-resolved graph before contacting the broker. */
|
|
16
|
+
export declare function planFlows<T>(flows: FlowJob<T>[], options?: FlowOpts): FlowPlan<T>;
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { generateJobId, jobId } from '../domain/types/job';
|
|
2
|
+
import { flowJobInput } from './flowOptions';
|
|
3
|
+
const MAX_FLOW_DEPTH = 100;
|
|
4
|
+
const MAX_FLOW_JOBS = 10_000;
|
|
5
|
+
function validateNodeShape(node, depth) {
|
|
6
|
+
if (!node || typeof node !== 'object')
|
|
7
|
+
throw new Error('flow node must be an object');
|
|
8
|
+
if (!node.name || typeof node.name !== 'string')
|
|
9
|
+
throw new Error('flow job name is required');
|
|
10
|
+
if (!node.queueName || typeof node.queueName !== 'string') {
|
|
11
|
+
throw new Error('flow queueName is required');
|
|
12
|
+
}
|
|
13
|
+
if (depth > MAX_FLOW_DEPTH) {
|
|
14
|
+
throw new Error(`flow exceeds the ${MAX_FLOW_DEPTH} level depth limit`);
|
|
15
|
+
}
|
|
16
|
+
if (node.children !== undefined && !Array.isArray(node.children)) {
|
|
17
|
+
throw new Error('flow children must be an array');
|
|
18
|
+
}
|
|
19
|
+
if (node.data !== undefined) {
|
|
20
|
+
if (typeof node.data !== 'object' || node.data === null || Array.isArray(node.data)) {
|
|
21
|
+
throw new Error('flow job data must be an object');
|
|
22
|
+
}
|
|
23
|
+
for (const key of Object.keys(node.data)) {
|
|
24
|
+
if (key === 'name' || key.startsWith('__')) {
|
|
25
|
+
throw new Error(`flow job data key is reserved: ${key}`);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
function plannedId(node) {
|
|
31
|
+
const custom = node.opts?.jobId;
|
|
32
|
+
if (custom === undefined)
|
|
33
|
+
return generateJobId();
|
|
34
|
+
if (!custom || custom.includes(':')) {
|
|
35
|
+
throw new Error('flow jobId must be non-empty and cannot contain a colon');
|
|
36
|
+
}
|
|
37
|
+
return jobId(custom);
|
|
38
|
+
}
|
|
39
|
+
/** Compile one or more trees into a fully-resolved graph before contacting the broker. */
|
|
40
|
+
export function planFlows(flows, options) {
|
|
41
|
+
const jobs = [];
|
|
42
|
+
const ids = new Set();
|
|
43
|
+
const seen = new WeakSet();
|
|
44
|
+
let nodeCount = 0;
|
|
45
|
+
const visit = (node, parent, depth) => {
|
|
46
|
+
validateNodeShape(node, depth);
|
|
47
|
+
if (seen.has(node))
|
|
48
|
+
throw new Error('flow contains a cycle or shared node');
|
|
49
|
+
seen.add(node);
|
|
50
|
+
nodeCount++;
|
|
51
|
+
if (nodeCount > MAX_FLOW_JOBS) {
|
|
52
|
+
throw new Error(`flow exceeds the ${MAX_FLOW_JOBS} job limit`);
|
|
53
|
+
}
|
|
54
|
+
const id = plannedId(node);
|
|
55
|
+
if (ids.has(String(id)))
|
|
56
|
+
throw new Error(`duplicate flow job id: ${String(id)}`);
|
|
57
|
+
ids.add(String(id));
|
|
58
|
+
const children = (node.children ?? []).map((child) => visit(child, { id, queue: node.queueName }, depth + 1));
|
|
59
|
+
const childIds = children.map((child) => child.id);
|
|
60
|
+
const defaults = options?.queuesOptions?.[node.queueName];
|
|
61
|
+
const opts = defaults ? { ...defaults, ...node.opts } : (node.opts ?? {});
|
|
62
|
+
const internalData = {
|
|
63
|
+
...node.data,
|
|
64
|
+
name: node.name,
|
|
65
|
+
};
|
|
66
|
+
if (parent) {
|
|
67
|
+
internalData.__parentId = String(parent.id);
|
|
68
|
+
internalData.__parentQueue = parent.queue;
|
|
69
|
+
}
|
|
70
|
+
if (childIds.length > 0)
|
|
71
|
+
internalData.__childrenIds = childIds.map(String);
|
|
72
|
+
jobs.push({
|
|
73
|
+
id,
|
|
74
|
+
queue: node.queueName,
|
|
75
|
+
input: flowJobInput(internalData, opts, {
|
|
76
|
+
parentId: parent?.id,
|
|
77
|
+
dependsOn: childIds.length > 0 ? childIds : undefined,
|
|
78
|
+
childrenIds: childIds.length > 0 ? childIds : undefined,
|
|
79
|
+
}),
|
|
80
|
+
});
|
|
81
|
+
return {
|
|
82
|
+
id,
|
|
83
|
+
name: node.name,
|
|
84
|
+
queueName: node.queueName,
|
|
85
|
+
data: node.data,
|
|
86
|
+
children: children.length > 0 ? children : undefined,
|
|
87
|
+
};
|
|
88
|
+
};
|
|
89
|
+
return {
|
|
90
|
+
batch: { jobs },
|
|
91
|
+
roots: flows.map((flow) => visit(flow, undefined, 0)),
|
|
92
|
+
};
|
|
93
|
+
}
|
package/dist/client/flowPush.js
CHANGED
|
@@ -169,7 +169,10 @@ export async function pushJobWithParent(ctx, params) {
|
|
|
169
169
|
// Update children with real parent ID (like embedded mode does)
|
|
170
170
|
if (childIds.length > 0) {
|
|
171
171
|
for (const childId of childIds) {
|
|
172
|
-
await ctx.tcp.send({ cmd: 'UpdateParent', childId, parentId: parentJobId });
|
|
172
|
+
const update = await ctx.tcp.send({ cmd: 'UpdateParent', childId, parentId: parentJobId });
|
|
173
|
+
if (update.ok !== true) {
|
|
174
|
+
throw new Error(typeof update.error === 'string' ? update.error : `Failed to link child ${childId}`);
|
|
175
|
+
}
|
|
173
176
|
}
|
|
174
177
|
}
|
|
175
178
|
return parentJobId;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { FlowJobCallbacks } from './flowJobTypes';
|
|
2
|
+
import type { GetFlowOpts, JobNode } from './flowTypes';
|
|
3
|
+
import type { TcpConnectionPool } from './tcpPool';
|
|
4
|
+
export interface FlowReaderContext {
|
|
5
|
+
embedded: boolean;
|
|
6
|
+
tcp: TcpConnectionPool | null;
|
|
7
|
+
buildCallbacks: (queueName: string) => FlowJobCallbacks;
|
|
8
|
+
}
|
|
9
|
+
/** Read a flow without silently truncating transport or topology errors. */
|
|
10
|
+
export declare function readFlow<T>(context: FlowReaderContext, opts: GetFlowOpts): Promise<JobNode<T> | null>;
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
import { jobId } from '../domain/types/job';
|
|
2
|
+
import { createFlowJobObject, extractUserDataFromInternal } from './flowJobFactory';
|
|
3
|
+
import { getSharedManager } from './manager';
|
|
4
|
+
function traversalLimit(value, name) {
|
|
5
|
+
if (value === undefined)
|
|
6
|
+
return Infinity;
|
|
7
|
+
if (value === Infinity)
|
|
8
|
+
return value;
|
|
9
|
+
if (!Number.isInteger(value) || value < 0) {
|
|
10
|
+
throw new Error(`${name} must be a non-negative integer`);
|
|
11
|
+
}
|
|
12
|
+
return value;
|
|
13
|
+
}
|
|
14
|
+
function childIdsFromTcp(job) {
|
|
15
|
+
const data = typeof job.data === 'object' && job.data !== null ? job.data : {};
|
|
16
|
+
const raw = job.childrenIds ?? data.__childrenIds;
|
|
17
|
+
if (raw === undefined || raw === null)
|
|
18
|
+
return [];
|
|
19
|
+
if (!Array.isArray(raw) || raw.some((id) => typeof id !== 'string')) {
|
|
20
|
+
throw new Error(`Flow job ${String(job.id)} has invalid childrenIds`);
|
|
21
|
+
}
|
|
22
|
+
return raw;
|
|
23
|
+
}
|
|
24
|
+
function assertNoCycle(id, ancestors) {
|
|
25
|
+
if (ancestors.has(id))
|
|
26
|
+
throw new Error(`Flow topology contains a cycle at job ${id}`);
|
|
27
|
+
const next = new Set(ancestors);
|
|
28
|
+
next.add(id);
|
|
29
|
+
return next;
|
|
30
|
+
}
|
|
31
|
+
function assertParentLink(id, parentId, data, expected) {
|
|
32
|
+
if (!expected)
|
|
33
|
+
return;
|
|
34
|
+
if (String(parentId) !== expected.id ||
|
|
35
|
+
data.__parentId !== expected.id ||
|
|
36
|
+
data.__parentQueue !== expected.queue) {
|
|
37
|
+
throw new Error(`Flow child ${id} does not point back to parent ${expected.id}`);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
async function buildEmbeddedNode(context, job, traversal) {
|
|
41
|
+
const { depth, maxChildren, ancestors, expectedParent } = traversal;
|
|
42
|
+
const id = String(job.id);
|
|
43
|
+
const nextAncestors = assertNoCycle(id, ancestors);
|
|
44
|
+
const data = job.data;
|
|
45
|
+
assertParentLink(id, job.parentId, data, expectedParent);
|
|
46
|
+
const name = typeof data.name === 'string' ? data.name : 'default';
|
|
47
|
+
const node = {
|
|
48
|
+
job: createFlowJobObject(id, name, extractUserDataFromInternal(data), job.queue, {
|
|
49
|
+
callbacks: context.buildCallbacks(job.queue),
|
|
50
|
+
snapshot: job,
|
|
51
|
+
}),
|
|
52
|
+
};
|
|
53
|
+
if (depth <= 0 || job.childrenIds.length === 0 || maxChildren === 0)
|
|
54
|
+
return node;
|
|
55
|
+
const childIds = maxChildren === Infinity ? job.childrenIds : job.childrenIds.slice(0, maxChildren);
|
|
56
|
+
const children = [];
|
|
57
|
+
for (const childId of childIds) {
|
|
58
|
+
const child = await getSharedManager().getJob(childId);
|
|
59
|
+
if (!child)
|
|
60
|
+
throw new Error(`Flow child job ${String(childId)} not found`);
|
|
61
|
+
children.push(await buildEmbeddedNode(context, child, {
|
|
62
|
+
depth: depth - 1,
|
|
63
|
+
maxChildren,
|
|
64
|
+
ancestors: nextAncestors,
|
|
65
|
+
expectedParent: { id, queue: job.queue },
|
|
66
|
+
}));
|
|
67
|
+
}
|
|
68
|
+
if (children.length > 0)
|
|
69
|
+
node.children = children;
|
|
70
|
+
return node;
|
|
71
|
+
}
|
|
72
|
+
async function getTcpJob(tcp, id, root) {
|
|
73
|
+
const response = await tcp.send({ cmd: 'GetJob', id });
|
|
74
|
+
if (response.ok !== true) {
|
|
75
|
+
if (root && response.error === 'Job not found')
|
|
76
|
+
return null;
|
|
77
|
+
throw new Error(typeof response.error === 'string' ? response.error : `Unable to fetch flow job ${id}`);
|
|
78
|
+
}
|
|
79
|
+
if (!response.job || typeof response.job !== 'object') {
|
|
80
|
+
throw new Error(`GetJob returned no snapshot for flow job ${id}`);
|
|
81
|
+
}
|
|
82
|
+
return response.job;
|
|
83
|
+
}
|
|
84
|
+
async function buildTcpNode(context, job, traversal) {
|
|
85
|
+
const { depth, maxChildren, ancestors, expectedParent } = traversal;
|
|
86
|
+
const tcp = context.tcp;
|
|
87
|
+
if (!tcp)
|
|
88
|
+
throw new Error('TCP connection not initialized');
|
|
89
|
+
const id = String(job.id);
|
|
90
|
+
const queueName = String(job.queue);
|
|
91
|
+
const nextAncestors = assertNoCycle(id, ancestors);
|
|
92
|
+
const data = typeof job.data === 'object' && job.data !== null ? job.data : {};
|
|
93
|
+
assertParentLink(id, job.parentId, data, expectedParent);
|
|
94
|
+
const name = typeof data.name === 'string' ? data.name : 'default';
|
|
95
|
+
const node = {
|
|
96
|
+
job: createFlowJobObject(id, name, extractUserDataFromInternal(data), queueName, {
|
|
97
|
+
callbacks: context.buildCallbacks(queueName),
|
|
98
|
+
snapshot: job,
|
|
99
|
+
}),
|
|
100
|
+
};
|
|
101
|
+
const childIds = childIdsFromTcp(job);
|
|
102
|
+
if (depth <= 0 || childIds.length === 0 || maxChildren === 0)
|
|
103
|
+
return node;
|
|
104
|
+
const selected = maxChildren === Infinity ? childIds : childIds.slice(0, maxChildren);
|
|
105
|
+
const children = [];
|
|
106
|
+
for (const childId of selected) {
|
|
107
|
+
const child = await getTcpJob(tcp, childId, false);
|
|
108
|
+
if (!child)
|
|
109
|
+
throw new Error(`Flow child job ${childId} not found`);
|
|
110
|
+
children.push(await buildTcpNode(context, child, {
|
|
111
|
+
depth: depth - 1,
|
|
112
|
+
maxChildren,
|
|
113
|
+
ancestors: nextAncestors,
|
|
114
|
+
expectedParent: { id, queue: queueName },
|
|
115
|
+
}));
|
|
116
|
+
}
|
|
117
|
+
if (children.length > 0)
|
|
118
|
+
node.children = children;
|
|
119
|
+
return node;
|
|
120
|
+
}
|
|
121
|
+
/** Read a flow without silently truncating transport or topology errors. */
|
|
122
|
+
export async function readFlow(context, opts) {
|
|
123
|
+
const depth = traversalLimit(opts.depth, 'depth');
|
|
124
|
+
const maxChildren = traversalLimit(opts.maxChildren, 'maxChildren');
|
|
125
|
+
if (context.embedded) {
|
|
126
|
+
const job = await getSharedManager().getJob(jobId(opts.id));
|
|
127
|
+
if (!job || job.queue !== opts.queueName)
|
|
128
|
+
return null;
|
|
129
|
+
return buildEmbeddedNode(context, job, {
|
|
130
|
+
depth,
|
|
131
|
+
maxChildren,
|
|
132
|
+
ancestors: new Set(),
|
|
133
|
+
expectedParent: null,
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
if (!context.tcp)
|
|
137
|
+
throw new Error('TCP connection not initialized');
|
|
138
|
+
const job = await getTcpJob(context.tcp, opts.id, true);
|
|
139
|
+
if (!job || job.queue !== opts.queueName)
|
|
140
|
+
return null;
|
|
141
|
+
return buildTcpNode(context, job, {
|
|
142
|
+
depth,
|
|
143
|
+
maxChildren,
|
|
144
|
+
ancestors: new Set(),
|
|
145
|
+
expectedParent: null,
|
|
146
|
+
});
|
|
147
|
+
}
|
|
@@ -81,6 +81,8 @@ export function buildJobOpts(job) {
|
|
|
81
81
|
sizeLimit: job.sizeLimit ?? undefined,
|
|
82
82
|
failParentOnFailure: job.failParentOnFailure,
|
|
83
83
|
removeDependencyOnFailure: job.removeDependencyOnFailure,
|
|
84
|
+
continueParentOnFailure: job.continueParentOnFailure,
|
|
85
|
+
ignoreDependencyOnFailure: job.ignoreDependencyOnFailure,
|
|
84
86
|
deduplication: job.deduplicationTtl !== null
|
|
85
87
|
? { id: job.customId ?? '', ttl: job.deduplicationTtl }
|
|
86
88
|
: undefined,
|
|
@@ -24,8 +24,10 @@ export interface Clock {
|
|
|
24
24
|
/** Schedule `fn` after `ms`. The handle is whatever `clear` accepts. */
|
|
25
25
|
setTimeout(fn: () => void, ms: number): TimerHandle;
|
|
26
26
|
clearTimeout(handle: TimerHandle): void;
|
|
27
|
-
/** A number in [0, 1). Used for retry jitter
|
|
27
|
+
/** A number in [0, 1). Used for retry jitter. */
|
|
28
28
|
random(): number;
|
|
29
|
+
/** Replayable entropy; cryptographically strong on the real clock. */
|
|
30
|
+
randomBytes(length: number): Uint8Array;
|
|
29
31
|
}
|
|
30
32
|
export interface TimerHandle {
|
|
31
33
|
/** Real timers can be unref'd so a pending one does not hold the process open. */
|
|
@@ -23,6 +23,7 @@ const realClock = {
|
|
|
23
23
|
setTimeout: (fn, ms) => globalThis.setTimeout(fn, ms),
|
|
24
24
|
clearTimeout: (h) => globalThis.clearTimeout(h),
|
|
25
25
|
random: () => Math.random(),
|
|
26
|
+
randomBytes: (length) => crypto.getRandomValues(new Uint8Array(length)),
|
|
26
27
|
};
|
|
27
28
|
let current = realClock;
|
|
28
29
|
/** The clock the engine reads. Real unless a simulation installed its own. */
|
|
@@ -58,6 +59,7 @@ export function simulatedClock(seed, startAt = 1_700_000_000_000) {
|
|
|
58
59
|
let time = startAt;
|
|
59
60
|
let sequence = 0;
|
|
60
61
|
let state = seed >>> 0 || 0x9e3779b9;
|
|
62
|
+
let entropyState = (seed ^ 0xa5a5a5a5) >>> 0 || 0x6d2b79f5;
|
|
61
63
|
const timers = new Map();
|
|
62
64
|
const random = () => {
|
|
63
65
|
// xorshift32: small, seeded, and identical across runs and platforms.
|
|
@@ -69,10 +71,25 @@ export function simulatedClock(seed, startAt = 1_700_000_000_000) {
|
|
|
69
71
|
return {
|
|
70
72
|
now: () => time,
|
|
71
73
|
random,
|
|
74
|
+
randomBytes(length) {
|
|
75
|
+
const bytes = new Uint8Array(length);
|
|
76
|
+
for (let i = 0; i < length; i++) {
|
|
77
|
+
entropyState ^= entropyState << 13;
|
|
78
|
+
entropyState ^= entropyState >>> 17;
|
|
79
|
+
entropyState ^= entropyState << 5;
|
|
80
|
+
bytes[i] = entropyState >>> 24;
|
|
81
|
+
}
|
|
82
|
+
return bytes;
|
|
83
|
+
},
|
|
72
84
|
setTimeout(fn, ms) {
|
|
73
85
|
const id = sequence++;
|
|
74
86
|
timers.set(id, { at: time + Math.max(0, ms), order: id, fn });
|
|
75
|
-
return {
|
|
87
|
+
return {
|
|
88
|
+
unref() {
|
|
89
|
+
// Simulated timers own no process handle.
|
|
90
|
+
},
|
|
91
|
+
__id: id,
|
|
92
|
+
};
|
|
76
93
|
},
|
|
77
94
|
clearTimeout(handle) {
|
|
78
95
|
const id = handle?.__id;
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/** Nested-saga compensation. */
|
|
2
|
+
import type { WorkflowEmitter } from './emitter';
|
|
3
|
+
import type { WorkflowStore } from './store';
|
|
4
|
+
import type { Execution, StepRecord } from './types';
|
|
5
|
+
import type { Workflow } from './workflow';
|
|
6
|
+
export interface ChildUnwindRequest {
|
|
7
|
+
exec: Execution;
|
|
8
|
+
workflow: Workflow;
|
|
9
|
+
store: WorkflowStore;
|
|
10
|
+
emitter: WorkflowEmitter | null;
|
|
11
|
+
workflows?: Map<string, Workflow>;
|
|
12
|
+
retryFailed?: boolean;
|
|
13
|
+
}
|
|
14
|
+
export type ChildUnwind = (request: ChildUnwindRequest) => Promise<'ran' | 'claim-lost'>;
|
|
15
|
+
export interface UnwindChildRequest {
|
|
16
|
+
record: StepRecord;
|
|
17
|
+
store: WorkflowStore;
|
|
18
|
+
emitter: WorkflowEmitter | null;
|
|
19
|
+
workflows?: Map<string, Workflow>;
|
|
20
|
+
runChild: ChildUnwind;
|
|
21
|
+
retryFailed?: boolean;
|
|
22
|
+
}
|
|
23
|
+
/** Run the child workflow's own unwind and propagate every uncertain outcome. */
|
|
24
|
+
export declare function unwindChild(request: UnwindChildRequest): Promise<void>;
|
|
25
|
+
export declare class AbandonedCompensationError extends Error {
|
|
26
|
+
constructor(workflowName: string, executionId: string);
|
|
27
|
+
}
|
|
28
|
+
export declare class CommittedChildError extends Error {
|
|
29
|
+
constructor(workflowName: string, executionId: string);
|
|
30
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/** Nested-saga compensation. */
|
|
2
|
+
import { isLive } from './admission';
|
|
3
|
+
/** Run the child workflow's own unwind and propagate every uncertain outcome. */
|
|
4
|
+
export async function unwindChild(request) {
|
|
5
|
+
const { record, store, emitter, workflows, runChild, retryFailed } = request;
|
|
6
|
+
const childId = record.childExecutionId;
|
|
7
|
+
if (!childId)
|
|
8
|
+
throw new Error('sub-workflow record carries no child execution id');
|
|
9
|
+
if (!workflows)
|
|
10
|
+
throw new Error('workflow registry unavailable to unwind a sub-workflow');
|
|
11
|
+
const child = store.get(childId);
|
|
12
|
+
if (!child)
|
|
13
|
+
throw new Error(`child execution "${childId}" not found`);
|
|
14
|
+
const childWf = workflows.get(child.workflowName);
|
|
15
|
+
if (!childWf)
|
|
16
|
+
throw new Error(`child workflow "${child.workflowName}" is not registered`);
|
|
17
|
+
if (child.state === 'failed' && child.rollbackStatus === 'stuck') {
|
|
18
|
+
throw new AbandonedCompensationError(child.workflowName, childId);
|
|
19
|
+
}
|
|
20
|
+
// Forward progress and rollback must never write the same child concurrently.
|
|
21
|
+
if (isLive(child.state)) {
|
|
22
|
+
throw new Error(`child "${child.workflowName}" (${childId}) is still ${child.state}; ` +
|
|
23
|
+
'it cannot be rolled back until it stops');
|
|
24
|
+
}
|
|
25
|
+
const outcome = await runChild({
|
|
26
|
+
exec: child,
|
|
27
|
+
workflow: childWf,
|
|
28
|
+
store,
|
|
29
|
+
emitter,
|
|
30
|
+
workflows,
|
|
31
|
+
retryFailed,
|
|
32
|
+
});
|
|
33
|
+
if (outcome === 'claim-lost') {
|
|
34
|
+
throw new Error(`child "${child.workflowName}" (${childId}) is being rolled back by another driver; ` +
|
|
35
|
+
'outcome unknown');
|
|
36
|
+
}
|
|
37
|
+
// The recursive unwind may have crossed an async boundary or been driven by
|
|
38
|
+
// another engine. Interpret the authoritative row, not the snapshot passed in.
|
|
39
|
+
const settledChild = store.get(childId);
|
|
40
|
+
if (!settledChild)
|
|
41
|
+
throw new Error(`child execution "${childId}" disappeared during rollback`);
|
|
42
|
+
if (settledChild.state === 'failed' && settledChild.rollbackStatus === 'stuck') {
|
|
43
|
+
throw new AbandonedCompensationError(settledChild.workflowName, childId);
|
|
44
|
+
}
|
|
45
|
+
if (settledChild.rollbackStatus === 'stuck') {
|
|
46
|
+
throw new Error(`child "${settledChild.workflowName}" (${childId}) parked mid-rollback`);
|
|
47
|
+
}
|
|
48
|
+
if (settledChild.rollbackStatus === 'not-applicable' && settledChild.committedAt !== undefined) {
|
|
49
|
+
throw new CommittedChildError(settledChild.workflowName, childId);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
export class AbandonedCompensationError extends Error {
|
|
53
|
+
constructor(workflowName, executionId) {
|
|
54
|
+
super(`child "${workflowName}" (${executionId}) has an explicitly abandoned rollback; ` +
|
|
55
|
+
'it is terminal and cannot be resumed through an ancestor');
|
|
56
|
+
this.name = 'AbandonedCompensationError';
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
export class CommittedChildError extends Error {
|
|
60
|
+
constructor(workflowName, executionId) {
|
|
61
|
+
super(`child "${workflowName}" (${executionId}) is committed past its pivot; ` +
|
|
62
|
+
'nothing was rolled back');
|
|
63
|
+
this.name = 'CommittedChildError';
|
|
64
|
+
}
|
|
65
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/** One checkpointed compensation pass. */
|
|
2
|
+
import { type ChildUnwind } from './compensationChild';
|
|
3
|
+
import type { WorkflowEmitter } from './emitter';
|
|
4
|
+
import type { WorkflowStore } from './store';
|
|
5
|
+
import type { Execution, StepRecord } from './types';
|
|
6
|
+
import type { Workflow } from './workflow';
|
|
7
|
+
export interface UnwindPass {
|
|
8
|
+
exec: Execution;
|
|
9
|
+
wf: Workflow;
|
|
10
|
+
store: WorkflowStore;
|
|
11
|
+
emitter: WorkflowEmitter | null;
|
|
12
|
+
eligible: [string, StepRecord][];
|
|
13
|
+
workflows?: Map<string, Workflow>;
|
|
14
|
+
retryFailed?: boolean;
|
|
15
|
+
}
|
|
16
|
+
export declare function unwind(pass: UnwindPass, runChild: ChildUnwind): Promise<void>;
|