@nestjs/workflows 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +23 -0
- package/README.md +50 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +8 -0
- package/dist/core/codecs/aes-gcm-payload.codec.d.ts +42 -0
- package/dist/core/codecs/aes-gcm-payload.codec.js +119 -0
- package/dist/core/codecs/index.d.ts +2 -0
- package/dist/core/codecs/index.js +2 -0
- package/dist/core/codecs/payload-codecs.d.ts +59 -0
- package/dist/core/codecs/payload-codecs.js +130 -0
- package/dist/core/index.d.ts +16 -0
- package/dist/core/index.js +21 -0
- package/dist/core/interfaces/clock.interface.d.ts +18 -0
- package/dist/core/interfaces/clock.interface.js +1 -0
- package/dist/core/interfaces/index.d.ts +9 -0
- package/dist/core/interfaces/index.js +9 -0
- package/dist/core/interfaces/leased-worker-options.interface.d.ts +94 -0
- package/dist/core/interfaces/leased-worker-options.interface.js +1 -0
- package/dist/core/interfaces/limits.interface.d.ts +92 -0
- package/dist/core/interfaces/limits.interface.js +1 -0
- package/dist/core/interfaces/payload-codec.interface.d.ts +64 -0
- package/dist/core/interfaces/payload-codec.interface.js +1 -0
- package/dist/core/interfaces/result-waiter-options.interface.d.ts +52 -0
- package/dist/core/interfaces/result-waiter-options.interface.js +1 -0
- package/dist/core/interfaces/retry-settings.interface.d.ts +71 -0
- package/dist/core/interfaces/retry-settings.interface.js +1 -0
- package/dist/core/interfaces/schedule-store.interface.d.ts +143 -0
- package/dist/core/interfaces/schedule-store.interface.js +1 -0
- package/dist/core/interfaces/schedule.interface.d.ts +106 -0
- package/dist/core/interfaces/schedule.interface.js +1 -0
- package/dist/core/interfaces/scheduler-options.interface.d.ts +152 -0
- package/dist/core/interfaces/scheduler-options.interface.js +1 -0
- package/dist/core/limits/index.d.ts +1 -0
- package/dist/core/limits/index.js +1 -0
- package/dist/core/limits/limits.d.ts +61 -0
- package/dist/core/limits/limits.js +130 -0
- package/dist/core/results/index.d.ts +1 -0
- package/dist/core/results/index.js +1 -0
- package/dist/core/results/result-waiter.d.ts +54 -0
- package/dist/core/results/result-waiter.js +140 -0
- package/dist/core/retries/index.d.ts +1 -0
- package/dist/core/retries/index.js +1 -0
- package/dist/core/retries/retry.d.ts +60 -0
- package/dist/core/retries/retry.js +107 -0
- package/dist/core/scheduling/cron.util.d.ts +52 -0
- package/dist/core/scheduling/cron.util.js +262 -0
- package/dist/core/scheduling/encoded-schedule.store.d.ts +26 -0
- package/dist/core/scheduling/encoded-schedule.store.js +54 -0
- package/dist/core/scheduling/in-memory-schedule.store.d.ts +20 -0
- package/dist/core/scheduling/in-memory-schedule.store.js +90 -0
- package/dist/core/scheduling/index.d.ts +3 -0
- package/dist/core/scheduling/index.js +3 -0
- package/dist/core/scheduling/rrule.util.d.ts +55 -0
- package/dist/core/scheduling/rrule.util.js +371 -0
- package/dist/core/scheduling/schedule-spec.d.ts +57 -0
- package/dist/core/scheduling/schedule-spec.js +233 -0
- package/dist/core/scheduling/scheduler.d.ts +134 -0
- package/dist/core/scheduling/scheduler.js +450 -0
- package/dist/core/scheduling/time-zone.util.d.ts +18 -0
- package/dist/core/scheduling/time-zone.util.js +88 -0
- package/dist/core/time/clock.d.ts +28 -0
- package/dist/core/time/clock.js +35 -0
- package/dist/core/time/duration.d.ts +21 -0
- package/dist/core/time/duration.js +30 -0
- package/dist/core/time/index.d.ts +2 -0
- package/dist/core/time/index.js +2 -0
- package/dist/core/utils/canonical.util.d.ts +2 -0
- package/dist/core/utils/canonical.util.js +4 -0
- package/dist/core/utils/maybe.util.d.ts +8 -0
- package/dist/core/utils/maybe.util.js +14 -0
- package/dist/core/utils/payload-place.util.d.ts +10 -0
- package/dist/core/utils/payload-place.util.js +32 -0
- package/dist/core/workers/index.d.ts +1 -0
- package/dist/core/workers/index.js +1 -0
- package/dist/core/workers/leased-worker.d.ts +86 -0
- package/dist/core/workers/leased-worker.js +280 -0
- package/dist/cqrs/decorators/index.d.ts +2 -0
- package/dist/cqrs/decorators/index.js +2 -0
- package/dist/cqrs/decorators/signal-on.decorator.d.ts +18 -0
- package/dist/cqrs/decorators/signal-on.decorator.js +34 -0
- package/dist/cqrs/decorators/start-on.decorator.d.ts +18 -0
- package/dist/cqrs/decorators/start-on.decorator.js +49 -0
- package/dist/cqrs/index.d.ts +3 -0
- package/dist/cqrs/index.js +5 -0
- package/dist/cqrs/interfaces/index.d.ts +4 -0
- package/dist/cqrs/interfaces/index.js +4 -0
- package/dist/cqrs/interfaces/signal-on-options.interface.d.ts +22 -0
- package/dist/cqrs/interfaces/signal-on-options.interface.js +1 -0
- package/dist/cqrs/interfaces/start-on-options.interface.d.ts +23 -0
- package/dist/cqrs/interfaces/start-on-options.interface.js +1 -0
- package/dist/cqrs/interfaces/workflow-dispatcher-context.interface.d.ts +10 -0
- package/dist/cqrs/interfaces/workflow-dispatcher-context.interface.js +1 -0
- package/dist/cqrs/interfaces/workflow-event-route.interface.d.ts +20 -0
- package/dist/cqrs/interfaces/workflow-event-route.interface.js +1 -0
- package/dist/cqrs/utils/event-routes.util.d.ts +7 -0
- package/dist/cqrs/utils/event-routes.util.js +22 -0
- package/dist/cqrs/workflow-event.publisher.d.ts +52 -0
- package/dist/cqrs/workflow-event.publisher.js +257 -0
- package/dist/cqrs/workflow-events.explorer.d.ts +27 -0
- package/dist/cqrs/workflow-events.explorer.js +92 -0
- package/dist/cqrs/workflows-cqrs.module.d.ts +10 -0
- package/dist/cqrs/workflows-cqrs.module.js +27 -0
- package/dist/decorators/index.d.ts +1 -0
- package/dist/decorators/index.js +1 -0
- package/dist/decorators/workflow.decorator.d.ts +7 -0
- package/dist/decorators/workflow.decorator.js +52 -0
- package/dist/errors/child-workflow-failed.error.d.ts +13 -0
- package/dist/errors/child-workflow-failed.error.js +14 -0
- package/dist/errors/index.d.ts +13 -0
- package/dist/errors/index.js +13 -0
- package/dist/errors/non-retryable-step.error.d.ts +4 -0
- package/dist/errors/non-retryable-step.error.js +7 -0
- package/dist/errors/step-failed.error.d.ts +15 -0
- package/dist/errors/step-failed.error.js +16 -0
- package/dist/errors/step-timeout.error.d.ts +5 -0
- package/dist/errors/step-timeout.error.js +8 -0
- package/dist/errors/workflow-definition.error.d.ts +10 -0
- package/dist/errors/workflow-definition.error.js +13 -0
- package/dist/errors/workflow-failed.error.d.ts +20 -0
- package/dist/errors/workflow-failed.error.js +17 -0
- package/dist/errors/workflow-id-conflict.error.d.ts +6 -0
- package/dist/errors/workflow-id-conflict.error.js +9 -0
- package/dist/errors/workflow-interrupt.error.d.ts +32 -0
- package/dist/errors/workflow-interrupt.error.js +19 -0
- package/dist/errors/workflow-non-determinism.error.d.ts +4 -0
- package/dist/errors/workflow-non-determinism.error.js +7 -0
- package/dist/errors/workflow-not-found.error.d.ts +10 -0
- package/dist/errors/workflow-not-found.error.js +13 -0
- package/dist/errors/workflow-result-timeout.error.d.ts +11 -0
- package/dist/errors/workflow-result-timeout.error.js +13 -0
- package/dist/errors/workflow-state.error.d.ts +11 -0
- package/dist/errors/workflow-state.error.js +14 -0
- package/dist/errors/workflow.error.d.ts +7 -0
- package/dist/errors/workflow.error.js +7 -0
- package/dist/events/index.d.ts +3 -0
- package/dist/events/index.js +3 -0
- package/dist/events/workflow-events.interface.d.ts +145 -0
- package/dist/events/workflow-events.interface.js +1 -0
- package/dist/events/workflow-events.service.d.ts +17 -0
- package/dist/events/workflow-events.service.js +38 -0
- package/dist/events/workflows.channels.d.ts +3 -0
- package/dist/events/workflows.channels.js +9 -0
- package/dist/index.d.ts +20 -0
- package/dist/index.js +23 -0
- package/dist/interfaces/index.d.ts +13 -0
- package/dist/interfaces/index.js +13 -0
- package/dist/interfaces/serialized-workflow-error.interface.d.ts +6 -0
- package/dist/interfaces/serialized-workflow-error.interface.js +1 -0
- package/dist/interfaces/workflow-client.interface.d.ts +156 -0
- package/dist/interfaces/workflow-client.interface.js +1 -0
- package/dist/interfaces/workflow-clock.interface.d.ts +7 -0
- package/dist/interfaces/workflow-clock.interface.js +1 -0
- package/dist/interfaces/workflow-context.interface.d.ts +241 -0
- package/dist/interfaces/workflow-context.interface.js +1 -0
- package/dist/interfaces/workflow-decorator-options.interface.d.ts +101 -0
- package/dist/interfaces/workflow-decorator-options.interface.js +1 -0
- package/dist/interfaces/workflow-instance.interface.d.ts +83 -0
- package/dist/interfaces/workflow-instance.interface.js +1 -0
- package/dist/interfaces/workflow-payload-codec.interface.d.ts +32 -0
- package/dist/interfaces/workflow-payload-codec.interface.js +1 -0
- package/dist/interfaces/workflow-retry-options.interface.d.ts +9 -0
- package/dist/interfaces/workflow-retry-options.interface.js +1 -0
- package/dist/interfaces/workflow-runner.interface.d.ts +11 -0
- package/dist/interfaces/workflow-runner.interface.js +1 -0
- package/dist/interfaces/workflow-schedule.interface.d.ts +104 -0
- package/dist/interfaces/workflow-schedule.interface.js +1 -0
- package/dist/interfaces/workflow-storage.interface.d.ts +4 -0
- package/dist/interfaces/workflow-storage.interface.js +1 -0
- package/dist/interfaces/workflow-store.interface.d.ts +488 -0
- package/dist/interfaces/workflow-store.interface.js +1 -0
- package/dist/interfaces/workflows-module-options.interface.d.ts +80 -0
- package/dist/interfaces/workflows-module-options.interface.js +1 -0
- package/dist/mysql/index.d.ts +5 -0
- package/dist/mysql/index.js +8 -0
- package/dist/mysql/interfaces/index.d.ts +1 -0
- package/dist/mysql/interfaces/index.js +1 -0
- package/dist/mysql/interfaces/mysql-workflow-store-options.interface.d.ts +25 -0
- package/dist/mysql/interfaces/mysql-workflow-store-options.interface.js +1 -0
- package/dist/mysql/key-limits.d.ts +30 -0
- package/dist/mysql/key-limits.js +41 -0
- package/dist/mysql/migrations/index.d.ts +7 -0
- package/dist/mysql/migrations/index.js +16 -0
- package/dist/mysql/migrations/initial.migration.d.ts +9 -0
- package/dist/mysql/migrations/initial.migration.js +104 -0
- package/dist/mysql/mysql-workflow.store.d.ts +197 -0
- package/dist/mysql/mysql-workflow.store.js +828 -0
- package/dist/postgres/errors/index.d.ts +1 -0
- package/dist/postgres/errors/index.js +2 -0
- package/dist/postgres/index.d.ts +5 -0
- package/dist/postgres/index.js +8 -0
- package/dist/postgres/interfaces/index.d.ts +1 -0
- package/dist/postgres/interfaces/index.js +1 -0
- package/dist/postgres/interfaces/postgres-workflow-store-options.interface.d.ts +22 -0
- package/dist/postgres/interfaces/postgres-workflow-store-options.interface.js +1 -0
- package/dist/postgres/migrations/index.d.ts +6 -0
- package/dist/postgres/migrations/index.js +15 -0
- package/dist/postgres/migrations/initial.migration.d.ts +6 -0
- package/dist/postgres/migrations/initial.migration.js +106 -0
- package/dist/postgres/postgres-workflow.store.d.ts +132 -0
- package/dist/postgres/postgres-workflow.store.js +586 -0
- package/dist/services/index.d.ts +6 -0
- package/dist/services/index.js +6 -0
- package/dist/services/workflow-client.service.d.ts +146 -0
- package/dist/services/workflow-client.service.js +436 -0
- package/dist/services/workflow-execution.service.d.ts +267 -0
- package/dist/services/workflow-execution.service.js +1076 -0
- package/dist/services/workflow-registry.service.d.ts +56 -0
- package/dist/services/workflow-registry.service.js +172 -0
- package/dist/services/workflow-scheduler.service.d.ts +54 -0
- package/dist/services/workflow-scheduler.service.js +173 -0
- package/dist/services/workflow-schedules.service.d.ts +60 -0
- package/dist/services/workflow-schedules.service.js +193 -0
- package/dist/services/workflow-worker.service.d.ts +92 -0
- package/dist/services/workflow-worker.service.js +468 -0
- package/dist/signals/index.d.ts +1 -0
- package/dist/signals/index.js +1 -0
- package/dist/signals/workflow.signal.d.ts +17 -0
- package/dist/signals/workflow.signal.js +28 -0
- package/dist/sql/rate-windows.d.ts +41 -0
- package/dist/sql/rate-windows.js +53 -0
- package/dist/sql/workflow-rows.d.ts +16 -0
- package/dist/sql/workflow-rows.js +102 -0
- package/dist/sql/workflow-schema.error.d.ts +21 -0
- package/dist/sql/workflow-schema.error.js +15 -0
- package/dist/storage/encoded-workflow.store.d.ts +71 -0
- package/dist/storage/encoded-workflow.store.js +201 -0
- package/dist/storage/index.d.ts +1 -0
- package/dist/storage/index.js +1 -0
- package/dist/storage/workflow.storage.d.ts +55 -0
- package/dist/storage/workflow.storage.js +159 -0
- package/dist/stores/in-memory-workflow.store.d.ts +75 -0
- package/dist/stores/in-memory-workflow.store.js +380 -0
- package/dist/stores/index.d.ts +1 -0
- package/dist/stores/index.js +1 -0
- package/dist/testing/assertions.d.ts +10 -0
- package/dist/testing/assertions.js +72 -0
- package/dist/testing/index.d.ts +38 -0
- package/dist/testing/index.js +1089 -0
- package/dist/testing/schedule-store.contract.d.ts +33 -0
- package/dist/testing/schedule-store.contract.js +161 -0
- package/dist/utils/index.d.ts +7 -0
- package/dist/utils/index.js +7 -0
- package/dist/utils/journal-limits.util.d.ts +6 -0
- package/dist/utils/journal-limits.util.js +25 -0
- package/dist/utils/new-instance.util.d.ts +26 -0
- package/dist/utils/new-instance.util.js +87 -0
- package/dist/utils/normalize.util.d.ts +5 -0
- package/dist/utils/normalize.util.js +11 -0
- package/dist/utils/run-timeout.util.d.ts +3 -0
- package/dist/utils/run-timeout.util.js +15 -0
- package/dist/utils/schedule-spec.util.d.ts +16 -0
- package/dist/utils/schedule-spec.util.js +9 -0
- package/dist/utils/schedule-store.util.d.ts +15 -0
- package/dist/utils/schedule-store.util.js +44 -0
- package/dist/utils/serialize-error.util.d.ts +7 -0
- package/dist/utils/serialize-error.util.js +22 -0
- package/dist/utils/step-scope.util.d.ts +16 -0
- package/dist/utils/step-scope.util.js +38 -0
- package/dist/workflows.constants.d.ts +10 -0
- package/dist/workflows.constants.js +10 -0
- package/dist/workflows.module-definition.d.ts +4 -0
- package/dist/workflows.module-definition.js +6 -0
- package/dist/workflows.module.d.ts +24 -0
- package/dist/workflows.module.js +68 -0
- package/package.json +123 -0
|
@@ -0,0 +1,1089 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { isDeepStrictEqual } from 'node:util';
|
|
3
|
+
import { absent, equal, expect, jitter, rejects } from './assertions.js';
|
|
4
|
+
import { scheduleCases } from './schedule-store.contract.js';
|
|
5
|
+
import { workflowScheduleStore } from '../utils/schedule-store.util.js';
|
|
6
|
+
export { scheduleStoreContract } from './schedule-store.contract.js';
|
|
7
|
+
/**
|
|
8
|
+
* The `WorkflowStore` contract as test cases, for any test runner:
|
|
9
|
+
*
|
|
10
|
+
* ```ts
|
|
11
|
+
* const cases = workflowStoreContract(async () => new DrizzleWorkflowStore(await freshDb(), new WorkflowStorage()), {
|
|
12
|
+
* concurrent: true,
|
|
13
|
+
* transaction: (work) => db.transaction(work),
|
|
14
|
+
* });
|
|
15
|
+
* for (const c of cases) it(c.name, c.run);
|
|
16
|
+
* ```
|
|
17
|
+
*
|
|
18
|
+
* `createStore` is called once per case and must return a store on empty tables.
|
|
19
|
+
*/
|
|
20
|
+
export function workflowStoreContract(createStore, options = {}) {
|
|
21
|
+
const cases = [];
|
|
22
|
+
const add = (name, run) => cases.push({
|
|
23
|
+
name,
|
|
24
|
+
run: async () => {
|
|
25
|
+
const store = await createStore();
|
|
26
|
+
try {
|
|
27
|
+
await run(new Harness(store));
|
|
28
|
+
}
|
|
29
|
+
finally {
|
|
30
|
+
await options.dispose?.(store);
|
|
31
|
+
}
|
|
32
|
+
},
|
|
33
|
+
});
|
|
34
|
+
const transaction = options.transaction;
|
|
35
|
+
// ---------------------------------------------------------------- instances
|
|
36
|
+
add('create() inserts a pending instance once, and returns the stored one for an existing id', async (t) => {
|
|
37
|
+
const input = { title: "O'Reilly — Designing Data-Intensive Applications ü 🚀", qty: 2, nested: [null, { a: true }], empty: '' };
|
|
38
|
+
const first = await t.store.create({ id: 'order-1', workflow: 'order-fulfilment', version: 3, input, deadline: null, now: 1_000 });
|
|
39
|
+
expect(first, {
|
|
40
|
+
created: true,
|
|
41
|
+
instance: {
|
|
42
|
+
id: 'order-1',
|
|
43
|
+
workflow: 'order-fulfilment',
|
|
44
|
+
version: 3,
|
|
45
|
+
parentId: null,
|
|
46
|
+
parentClose: null,
|
|
47
|
+
concurrencyKey: null,
|
|
48
|
+
rateLimitKey: null,
|
|
49
|
+
priority: 0,
|
|
50
|
+
scheduleId: null,
|
|
51
|
+
scheduledAt: null,
|
|
52
|
+
status: 'pending',
|
|
53
|
+
input,
|
|
54
|
+
error: null,
|
|
55
|
+
wakeAt: 1_000,
|
|
56
|
+
leaseOwner: null,
|
|
57
|
+
leaseUntil: null,
|
|
58
|
+
cancelRequested: false,
|
|
59
|
+
terminateRequested: false,
|
|
60
|
+
cancelReason: null,
|
|
61
|
+
deadline: null,
|
|
62
|
+
customStatus: null,
|
|
63
|
+
signalCursor: 0,
|
|
64
|
+
runs: 0,
|
|
65
|
+
createdAt: 1_000,
|
|
66
|
+
updatedAt: 1_000,
|
|
67
|
+
},
|
|
68
|
+
}, 'the new instance');
|
|
69
|
+
absent(first.instance.output, 'output of a new instance', { orNull: true });
|
|
70
|
+
const again = await t.store.create({ id: 'order-1', workflow: 'other', version: 1, input: 'different', deadline: null, now: 2_000 });
|
|
71
|
+
expect(again, { created: false, instance: first.instance }, 'the existing instance, unchanged');
|
|
72
|
+
expect(await t.store.get('order-1'), { ...first.instance, waits: [] }, 'get()');
|
|
73
|
+
equal(await t.store.get('missing'), null, 'get() of an unknown id');
|
|
74
|
+
equal((await t.store.create({ id: 'null-input', workflow: 'w', version: 1, input: null, deadline: null, now: 0 })).instance.input, null, 'a null input');
|
|
75
|
+
absent((await t.store.create({ id: 'no-input', workflow: 'w', version: 1, input: undefined, deadline: null, now: 0 })).instance.input, 'an undefined input', { orNull: true });
|
|
76
|
+
equal((await t.store.get('order-1', { journal: true }))?.journal, [], 'the journal of a new instance');
|
|
77
|
+
const timed = await t.store.create({ id: 'timed', workflow: 'w', version: 1, input: null, deadline: 86_401_000, now: 1_000 });
|
|
78
|
+
expect(timed.instance, { deadline: 86_401_000 }, 'a deadline');
|
|
79
|
+
expect(await t.store.get('timed'), { deadline: 86_401_000 }, 'a deadline, read back');
|
|
80
|
+
});
|
|
81
|
+
add('create() starts the signal cursor at the last signal sent before the instance', async (t) => {
|
|
82
|
+
await t.store.signal({ name: 's', key: null, dedupeId: null, payload: 1, now: 0 });
|
|
83
|
+
const { id } = await t.store.signal({ name: 's', key: null, dedupeId: null, payload: 2, now: 0 });
|
|
84
|
+
equal((await t.create('a')).instance.signalCursor, id, 'signalCursor');
|
|
85
|
+
equal((await t.claim(0)).lastSignalId, id, 'claim().lastSignalId');
|
|
86
|
+
});
|
|
87
|
+
add('list() filters by status, workflow and version, oldest first, a page at a time', async (t) => {
|
|
88
|
+
await t.create('c', 3);
|
|
89
|
+
await t.create('a', 1);
|
|
90
|
+
await t.create('b', 2, 'order-fulfilment', 2);
|
|
91
|
+
await t.create('d', 4, 'invoice-batch');
|
|
92
|
+
await t.create('e', 3); // same createdAt as 'c': ordered by id
|
|
93
|
+
await t.claim(1, { limit: 1 }); // 'a' is now running
|
|
94
|
+
const ids = async (query) => (await t.store.list({ limit: 100, offset: 0, ...query })).map((i) => i.id);
|
|
95
|
+
equal(await ids({}), ['a', 'b', 'c', 'e', 'd'], 'every instance');
|
|
96
|
+
equal(await ids({ status: ['running'] }), ['a'], 'by status');
|
|
97
|
+
equal(await ids({ status: ['pending', 'running'], limit: 2, offset: 1 }), ['b', 'c'], 'a page');
|
|
98
|
+
equal(await ids({ workflow: 'order-fulfilment', version: 1 }), ['a', 'c', 'e'], 'by workflow and version');
|
|
99
|
+
equal(await ids({ workflow: 'invoice-batch' }), ['d'], 'by workflow');
|
|
100
|
+
equal(await ids({ status: ['completed'] }), [], 'nothing matches');
|
|
101
|
+
});
|
|
102
|
+
add('create() keeps the parent link, and list() finds the children by parentId', async (t) => {
|
|
103
|
+
await t.create('order-1');
|
|
104
|
+
const child = (id, parentId, parentClose, now) => t.store.create({ id, workflow: 'shipping', version: 1, input: { id }, deadline: null, parentId, parentClose, now });
|
|
105
|
+
expect(await child('order-1/shipping#1', 'order-1', 'cancel', 5), { created: true, instance: { parentId: 'order-1', parentClose: 'cancel' } }, 'a child');
|
|
106
|
+
await child('order-1/shipping#2', 'order-1', 'abandon', 6);
|
|
107
|
+
await child('order-2/shipping#1', 'order-2', 'terminate', 4);
|
|
108
|
+
expect(await t.store.get('order-1/shipping#2'), { parentId: 'order-1', parentClose: 'abandon' }, 'read back');
|
|
109
|
+
expect(await t.store.get('order-1'), { parentId: null, parentClose: null }, 'no parent');
|
|
110
|
+
const children = async (parentId, extra = {}) => (await t.store.list({ limit: 100, offset: 0, parentId, ...extra })).map((i) => i.id);
|
|
111
|
+
equal(await children('order-1'), ['order-1/shipping#1', 'order-1/shipping#2'], "order-1's children, oldest first");
|
|
112
|
+
equal(await children('order-1', { status: ['pending'], limit: 1, offset: 1 }), ['order-1/shipping#2'], 'a page, with other filters');
|
|
113
|
+
equal(await children('order-2'), ['order-2/shipping#1'], "order-2's");
|
|
114
|
+
equal(await children('order-1/shipping#1'), [], 'none');
|
|
115
|
+
});
|
|
116
|
+
add("create() keeps the schedule link, and list() finds a schedule's instances by scheduleId", async (t) => {
|
|
117
|
+
const scheduled = (id, scheduleId, scheduledAt, now) => t.store.create({ id, workflow: W.name, version: 1, input: null, deadline: null, scheduleId, scheduledAt, now });
|
|
118
|
+
expect(await scheduled('digest@1', 'digest', 1_000, 5), { created: true, instance: { scheduleId: 'digest', scheduledAt: 1_000 } }, 'an occurrence');
|
|
119
|
+
await scheduled('digest@2', 'digest', 2_000, 6);
|
|
120
|
+
await scheduled('report@1', 'report', 1_000, 4);
|
|
121
|
+
await t.create('manual', 3);
|
|
122
|
+
expect(await t.store.get('digest@2'), { scheduleId: 'digest', scheduledAt: 2_000 }, 'read back');
|
|
123
|
+
const ids = async (scheduleId, extra = {}) => (await t.store.list({ limit: 100, offset: 0, scheduleId, ...extra })).map((i) => i.id);
|
|
124
|
+
equal(await ids('digest'), ['digest@1', 'digest@2'], "digest's, oldest first");
|
|
125
|
+
equal(await ids('digest', { status: ['pending'], limit: 1, offset: 1 }), ['digest@2'], 'a page, with other filters');
|
|
126
|
+
equal(await ids('none'), [], 'none');
|
|
127
|
+
});
|
|
128
|
+
// ---------------------------------------------------------------- claims and leases
|
|
129
|
+
add('claim() leases due, unleased instances of the given versions, most overdue first', async (t) => {
|
|
130
|
+
await t.create('late', 50);
|
|
131
|
+
await t.create('early', 10);
|
|
132
|
+
await t.create('v2', 5, 'order-fulfilment', 2);
|
|
133
|
+
await t.create('future', 5_000);
|
|
134
|
+
const first = await t.claim(100, { owner: 'w1', token: 't1' });
|
|
135
|
+
equal(first.instances.map((i) => i.id), ['early', 'late'], 'claimed, by wakeAt');
|
|
136
|
+
expect(first.instances[0], { status: 'running', leaseOwner: 'w1', leaseUntil: 1_100, runs: 1, updatedAt: 100 }, 'a claimed instance');
|
|
137
|
+
equal((await t.claim(1_100, { owner: 'w2' })).instances, [], 'leased until 1100 inclusive');
|
|
138
|
+
const retaken = await t.claim(1_101, { owner: 'w2', token: 't2' });
|
|
139
|
+
equal(retaken.instances.map((i) => [i.id, i.leaseOwner, i.runs]), [['early', 'w2', 2], ['late', 'w2', 2]], 'claimed again after the lease expired');
|
|
140
|
+
equal(await t.store.write('early', 't1', t.journalWrite([entry('x')])), false, 'a write under the first token');
|
|
141
|
+
equal(await t.store.write('early', 't2', t.journalWrite([entry('x')])), true, 'a write under the new token');
|
|
142
|
+
const v2 = await t.claim(2_000, { owner: 'w3', limit: 1, workflows: [{ name: 'order-fulfilment', version: 2 }, { name: 'invoice-batch', version: 1 }] });
|
|
143
|
+
equal(v2.instances.map((i) => i.id), ['v2'], 'only the versions the worker runs');
|
|
144
|
+
equal((await t.claim(6_000, { owner: 'w4', limit: 1 })).instances.map((i) => i.id), ['early'], 'at most limit');
|
|
145
|
+
});
|
|
146
|
+
add("claim() under a workflow's limit leases no more than it allows, counting the leases still live", async (t) => {
|
|
147
|
+
for (const [i, id] of ['a', 'b', 'c', 'd'].entries()) {
|
|
148
|
+
await t.keyed(id, null, i);
|
|
149
|
+
}
|
|
150
|
+
await t.create('x', 0, 'invoice-batch');
|
|
151
|
+
const limits = [{ workflow: W.name, limit: 2, perKey: null }];
|
|
152
|
+
const claim = (now, token) => t.claim(now, { token, limits, leaseUntil: now + 100, workflows: [W, { name: 'invoice-batch', version: 1 }] });
|
|
153
|
+
equal((await claim(10, 't1')).instances.map((i) => i.id), ['a', 'x', 'b'], 'two of the limited workflow, and the unlimited one');
|
|
154
|
+
equal((await claim(20, 't2')).instances.map((i) => i.id), [], 'both slots held');
|
|
155
|
+
equal(await t.release('a', 't1'), true, "a's lease ends");
|
|
156
|
+
equal((await claim(30, 't3')).instances.map((i) => i.id), ['c'], 'the slot it freed');
|
|
157
|
+
// b's lease (until 110) expired: b is due again, holds no slot, and goes before d. So is x's.
|
|
158
|
+
equal((await claim(111, 't4')).instances.map((i) => [i.id, i.runs]), [['x', 2], ['b', 2]], 'the slot an expired lease freed');
|
|
159
|
+
equal((await t.claim(111, { token: 't5' })).instances.map((i) => i.id), ['d'], 'a claim without limits');
|
|
160
|
+
});
|
|
161
|
+
add('claim() keeps at most perKey of a key leased, passes over a full key, and takes each key in order', async (t) => {
|
|
162
|
+
await t.keyed('c1-a', 'customer-1', 0);
|
|
163
|
+
await t.keyed('c1-b', 'customer-1', 1);
|
|
164
|
+
await t.keyed('c2-a', 'customer-2', 2);
|
|
165
|
+
await t.keyed('none', null, 3);
|
|
166
|
+
await t.keyed('c1-c', 'customer-1', 4);
|
|
167
|
+
await t.keyed('c3-a', 'customer-3', 5);
|
|
168
|
+
expect(await t.store.get('c1-a'), { concurrencyKey: 'customer-1' }, 'the key, stored');
|
|
169
|
+
const perKey = [{ workflow: W.name, limit: null, perKey: 1 }];
|
|
170
|
+
equal((await t.claim(10, { token: 't1', limits: perKey, limit: 3 })).instances.map((i) => i.id), ['c1-a', 'c2-a', 'none'], 'one per key, and one without a key');
|
|
171
|
+
equal((await t.claim(10, { token: 't2', limits: perKey })).instances.map((i) => i.id), ['c3-a'], 'the next free key');
|
|
172
|
+
equal((await t.claim(10, { token: 't3', limits: perKey })).instances.map((i) => i.id), [], 'every key full');
|
|
173
|
+
await t.release('c1-a', 't1');
|
|
174
|
+
equal((await t.claim(10, { token: 't4', limits: perKey })).instances.map((i) => i.id), ['c1-b'], "the key's next, not its last");
|
|
175
|
+
// Both limits: the workflow's counts every key's.
|
|
176
|
+
const both = [{ workflow: W.name, limit: 5, perKey: 1 }];
|
|
177
|
+
await t.keyed('c4-a', 'customer-4', 6);
|
|
178
|
+
await t.keyed('c5-a', 'customer-5', 7);
|
|
179
|
+
equal((await t.claim(10, { token: 't5', limits: both })).instances.map((i) => i.id), ['c4-a'], 'the fifth slot');
|
|
180
|
+
equal((await t.claim(10, { token: 't6', limits: both })).instances.map((i) => i.id), [], 'the workflow full');
|
|
181
|
+
});
|
|
182
|
+
add('claim() takes the lowest priority first (none before any), then the most overdue, with or without limits', async (t) => {
|
|
183
|
+
const instances = [
|
|
184
|
+
['late-none', 50, 0],
|
|
185
|
+
['p5', 0, 5],
|
|
186
|
+
['early-none', 10, 0],
|
|
187
|
+
['p1-late', 40, 1],
|
|
188
|
+
['p1-early', 20, 1],
|
|
189
|
+
];
|
|
190
|
+
for (const [id, now, priority] of instances) {
|
|
191
|
+
await t.prioritized(id, now, priority === 0 ? {} : { priority });
|
|
192
|
+
}
|
|
193
|
+
expect(await t.store.get('p5'), { priority: 5, rateLimitKey: null }, 'the priority, stored');
|
|
194
|
+
expect(await t.store.get('late-none'), { priority: 0 }, 'none given: 0');
|
|
195
|
+
const order = ['early-none', 'late-none', 'p1-early', 'p1-late', 'p5'];
|
|
196
|
+
equal(await t.claimAndFinish(100, { limit: 3 }), order.slice(0, 3), 'the first three');
|
|
197
|
+
equal(await t.claimAndFinish(100), order.slice(3), 'the rest');
|
|
198
|
+
// The same order through the limited path, one at a time.
|
|
199
|
+
for (const [id, now, priority] of instances) {
|
|
200
|
+
await t.prioritized(`k-${id}`, now, { priority });
|
|
201
|
+
}
|
|
202
|
+
const limits = [{ workflow: W.name, limit: 1, perKey: null }];
|
|
203
|
+
const taken = [];
|
|
204
|
+
for (let i = 0; i < order.length; i++) {
|
|
205
|
+
const [instance] = (await t.claim(200, { token: `l${i}`, limits, leaseUntil: 9_000 })).instances;
|
|
206
|
+
taken.push(instance?.id ?? 'nothing');
|
|
207
|
+
await t.release(instance?.id ?? 'nothing', `l${i}`);
|
|
208
|
+
}
|
|
209
|
+
equal(taken, order.map((id) => `k-${id}`), 'with a concurrency limit');
|
|
210
|
+
});
|
|
211
|
+
add("claim() under a workflow's rate limit starts at most max per window, and opens the next window at the first claim after it ended", async (t) => {
|
|
212
|
+
for (let i = 0; i < 7; i++) {
|
|
213
|
+
await t.prioritized(`r${i}`, i);
|
|
214
|
+
}
|
|
215
|
+
await t.create('x', 0, 'invoice-batch');
|
|
216
|
+
const rateLimits = [{ workflow: W.name, limit: { max: 3, duration: 1_000 }, perKey: null }];
|
|
217
|
+
const claim = (now, o = {}) => t.claimAndFinish(now, { rateLimits, workflows: [W, { name: 'invoice-batch', version: 1 }], ...o });
|
|
218
|
+
equal(await claim(100, { limit: 2 }), ['r0', 'x'], 'two, of which one of the unlimited workflow');
|
|
219
|
+
equal(await claim(200), ['r1', 'r2'], 'the rest of the window: three in all');
|
|
220
|
+
equal(await claim(300), [], 'the window is full');
|
|
221
|
+
equal(await claim(1_099), [], 'until it ends');
|
|
222
|
+
equal(await claim(1_100), ['r3', 'r4', 'r5'], 'a new window opens at the first claim after it ended');
|
|
223
|
+
equal(await claim(2_099), [], 'and ends a duration later');
|
|
224
|
+
equal(await claim(2_100), ['r6'], 'the next one');
|
|
225
|
+
});
|
|
226
|
+
add('claim() under per-key rate limits keeps each key to its window, passes over a full key, and counts keyless instances only toward the workflow', async (t) => {
|
|
227
|
+
await t.prioritized('a1', 0, { rateLimitKey: 'customer-a' });
|
|
228
|
+
await t.prioritized('a2', 1, { rateLimitKey: 'customer-a' });
|
|
229
|
+
await t.prioritized('b1', 2, { rateLimitKey: 'customer-b' });
|
|
230
|
+
await t.prioritized('none', 3);
|
|
231
|
+
await t.prioritized('a3', 4, { rateLimitKey: 'customer-a' });
|
|
232
|
+
await t.prioritized('c1', 5, { rateLimitKey: 'customer-c' });
|
|
233
|
+
expect(await t.store.get('a1'), { rateLimitKey: 'customer-a' }, 'the key, stored');
|
|
234
|
+
const perKey = [{ workflow: W.name, limit: null, perKey: { max: 1, duration: 1_000 } }];
|
|
235
|
+
equal(await t.claimAndFinish(10, { rateLimits: perKey, limit: 3 }), ['a1', 'b1', 'none'], 'one per key, and the keyless one');
|
|
236
|
+
equal(await t.claimAndFinish(20, { rateLimits: perKey }), ['c1'], 'the next key with room');
|
|
237
|
+
equal(await t.claimAndFinish(30, { rateLimits: perKey }), [], "customer-a's window is full");
|
|
238
|
+
equal(await t.claimAndFinish(1_010, { rateLimits: perKey }), ['a2'], "customer-a's window reopens with its next instance");
|
|
239
|
+
// Both: the workflow's window counts every key's claims.
|
|
240
|
+
for (const [i, key] of ['d', 'd', 'e', 'f'].entries()) {
|
|
241
|
+
await t.prioritized(`${key}${i}`, 100 + i, { rateLimitKey: `customer-${key}` });
|
|
242
|
+
}
|
|
243
|
+
const both = [{ workflow: W.name, limit: { max: 2, duration: 1_000 }, perKey: { max: 1, duration: 1_000 } }];
|
|
244
|
+
equal(await t.claimAndFinish(5_000, { rateLimits: both }), ['a3', 'd0'], 'two in all, one per key');
|
|
245
|
+
equal(await t.claimAndFinish(5_500, { rateLimits: both }), [], 'the workflow window is full');
|
|
246
|
+
equal(await t.claimAndFinish(6_000, { rateLimits: both }), ['d1', 'e2'], 'the next window');
|
|
247
|
+
});
|
|
248
|
+
add('claim() applies rate limits and concurrency limits together, in stages', async (t) => {
|
|
249
|
+
const keys = [
|
|
250
|
+
['i0', 'k0', 'r0'],
|
|
251
|
+
['i1', 'k0', 'r1'],
|
|
252
|
+
['i2', 'k1', 'r0'],
|
|
253
|
+
['i3', 'k1', 'r1'],
|
|
254
|
+
['i4', 'k2', 'r2'],
|
|
255
|
+
];
|
|
256
|
+
for (const [n, [id, concurrencyKey, rateLimitKey]] of keys.entries()) {
|
|
257
|
+
await t.store.create({ id, workflow: W.name, version: 1, input: null, deadline: null, now: n, concurrencyKey, rateLimitKey });
|
|
258
|
+
}
|
|
259
|
+
const limits = [{ workflow: W.name, limit: 3, perKey: 1 }];
|
|
260
|
+
const rateLimits = [{ workflow: W.name, limit: { max: 4, duration: 1_000 }, perKey: { max: 1, duration: 1_000 } }];
|
|
261
|
+
const claim = (now) => t.claim(now, { limits, rateLimits, leaseUntil: now + 10 }).then((c) => c.instances.map((i) => i.id));
|
|
262
|
+
// k1 keeps i2, its first, which r0 then drops for i0: i3 would fit, but waits for the next claim.
|
|
263
|
+
equal(await t.claimAndFinish(10, { limits, rateLimits }), ['i0', 'i4'], 'in stages');
|
|
264
|
+
// i2 has no room at all (r0 is full): passed over. k0 keeps i1, k1 keeps i3, and r1 keeps i1, the first.
|
|
265
|
+
equal(await t.claimAndFinish(20, { limits, rateLimits }), ['i1'], 'a full rate key passed over');
|
|
266
|
+
equal(await t.claimAndFinish(30, { limits, rateLimits }), [], 'r0 and r1 are full');
|
|
267
|
+
equal(await claim(1_010), ['i2'], 'in the next windows, k1 keeps i2');
|
|
268
|
+
equal(await claim(1_015), [], "k1's slot is held by i2's live lease");
|
|
269
|
+
equal(await claim(1_021), ['i3'], "i2's lease ended, but its r0 window is full; i3 takes k1's slot");
|
|
270
|
+
equal(await t.claimAndFinish(1_030, { limits, rateLimits }), [], "i2 waits for r0's next window, at 2010");
|
|
271
|
+
});
|
|
272
|
+
add('claim() keeps a compensating instance compensating, and never claims a finished one', async (t) => {
|
|
273
|
+
await t.create('a');
|
|
274
|
+
await t.create('b');
|
|
275
|
+
const { instances } = await t.claim(1, { token: 't1' });
|
|
276
|
+
equal(instances.length, 2, 'claimed');
|
|
277
|
+
await t.store.write('a', 't1', { now: 2, entries: [], status: 'compensating', error: { name: 'StepFailedError', message: 'no' } });
|
|
278
|
+
expect(await t.store.get('a'), { status: 'compensating', error: { name: 'StepFailedError', message: 'no' }, updatedAt: 2, leaseOwner: 'w1' }, 'compensating, lease kept');
|
|
279
|
+
await t.store.write('a', 't1', { now: 3, entries: [], release: { wakeAt: 4, waits: [], signalCursor: 0 } });
|
|
280
|
+
expect((await t.claim(4, { token: 't2' })).instances[0], { id: 'a', status: 'compensating', runs: 2 }, 'claimed again, still compensating');
|
|
281
|
+
await t.store.write('b', 't1', { now: 3, entries: [], status: 'completed', output: 1, error: null, release: { wakeAt: null, waits: [], signalCursor: 0 } });
|
|
282
|
+
equal((await t.claim(FAR, { token: 't3' })).instances.map((i) => i.id), ['a'], "'b' finished: never claimed again");
|
|
283
|
+
});
|
|
284
|
+
add('renew() extends the lease and reads the cancel flags while the token is current, and returns null after', async (t) => {
|
|
285
|
+
await t.create('a');
|
|
286
|
+
await t.claim(1, { token: 't1' });
|
|
287
|
+
equal(await t.store.renew('a', 't1', 5_000), { cancelRequested: false, terminateRequested: false }, 'renewed');
|
|
288
|
+
expect(await t.store.get('a'), { leaseUntil: 5_000 }, 'the new lease');
|
|
289
|
+
equal(await t.store.requestCancel('a', { reason: 'stop', now: 2, terminate: false }), true, 'cancel requested');
|
|
290
|
+
equal(await t.store.renew('a', 't1', 6_000), { cancelRequested: true, terminateRequested: false }, 'renew reads the flag');
|
|
291
|
+
equal(await t.store.requestCancel('a', { reason: 'now', now: 3, terminate: true }), true, 'terminate requested');
|
|
292
|
+
equal(await t.store.renew('a', 't1', 6_000), { cancelRequested: true, terminateRequested: true }, 'renew reads both flags');
|
|
293
|
+
equal(await t.store.renew('a', 'other', 7_000), null, 'another token');
|
|
294
|
+
equal(await t.store.renew('missing', 't1', 7_000), null, 'an unknown id');
|
|
295
|
+
expect(await t.store.get('a'), { leaseUntil: 6_000 }, 'unchanged by a stale renew');
|
|
296
|
+
});
|
|
297
|
+
add('write() changes nothing under a stale token', async (t) => {
|
|
298
|
+
await t.create('a');
|
|
299
|
+
await t.claim(1, { token: 'stale' });
|
|
300
|
+
await t.claim(5_000, { owner: 'w2', token: 'current' }); // the first lease expired, w2 took over
|
|
301
|
+
const writes = [
|
|
302
|
+
t.journalWrite([entry('x')]),
|
|
303
|
+
{ now: 1, entries: [], status: 'compensating', error: { name: 'Error', message: 'boom' } },
|
|
304
|
+
{ now: 1, entries: [entry('y')], status: 'suspended', release: { wakeAt: 9, waits: [{ signal: 's', key: null }], signalCursor: 0 } },
|
|
305
|
+
{ now: 1, entries: [], status: 'completed', output: 1, error: null, release: { wakeAt: null, waits: [], signalCursor: 0 } },
|
|
306
|
+
{ now: 1, entries: [entry('z')], release: { wakeAt: 1, waits: [], signalCursor: 0 } },
|
|
307
|
+
];
|
|
308
|
+
for (const [i, write] of writes.entries()) {
|
|
309
|
+
equal(await t.store.write('a', 'stale', write), false, `stale write #${i + 1}`);
|
|
310
|
+
}
|
|
311
|
+
equal(await t.store.write('missing', 'current', t.journalWrite([entry('x')])), false, 'a write to an unknown id');
|
|
312
|
+
expect(await t.store.get('a', { journal: true }), { status: 'running', leaseOwner: 'w2', leaseUntil: 6_000, error: null, waits: [], journal: [] }, 'nothing written');
|
|
313
|
+
});
|
|
314
|
+
add('write() keeps journal entries in first-write order, replaces them whole, and keeps null apart from undefined', async (t) => {
|
|
315
|
+
await t.create('a');
|
|
316
|
+
await t.claim(1, { token: 't' });
|
|
317
|
+
const progress = { cursor: 4, note: "it's 100% ü" };
|
|
318
|
+
await t.store.write('a', 't', t.journalWrite([entry('one', { status: 'pending', attempts: 1, wakeAt: null, progress })]));
|
|
319
|
+
await t.store.write('a', 't', t.journalWrite([
|
|
320
|
+
entry('two', { kind: 'signal', status: 'pending', attempts: 0, wakeAt: 99, data: { signal: 's', key: null } }),
|
|
321
|
+
entry('one', { status: 'failed', attempts: 2, error: { name: 'Error', message: 'x' }, wakeAt: 50 }),
|
|
322
|
+
entry('three', { kind: 'now', result: 123 }),
|
|
323
|
+
]));
|
|
324
|
+
await t.store.write('a', 't', t.journalWrite([
|
|
325
|
+
entry('four', { result: null }),
|
|
326
|
+
entry('two', { kind: 'signal', status: 'completed', attempts: 0, wakeAt: 99, result: { signalId: 7, payload: null }, data: { signal: 's', key: null } }),
|
|
327
|
+
]));
|
|
328
|
+
const journal = (await t.store.get('a', { journal: true })).journal;
|
|
329
|
+
equal(journal.map((e) => e.name), ['one', 'two', 'three', 'four'], 'first-write order');
|
|
330
|
+
expect(journal[0], { name: 'one', kind: 'step', status: 'failed', error: { name: 'Error', message: 'x' }, attempts: 2, wakeAt: 50, updatedAt: 1 }, "'one', replaced");
|
|
331
|
+
absent(journal[0].progress, "'one'.progress, dropped by the replacement");
|
|
332
|
+
expect(journal[1], { name: 'two', status: 'completed', result: { signalId: 7, payload: null }, data: { signal: 's', key: null } }, "'two'");
|
|
333
|
+
equal(journal[3].result, null, 'a null result stays null');
|
|
334
|
+
absent(journal[2].error, 'an undefined error stays undefined');
|
|
335
|
+
equal((await t.store.get('a')).journal, undefined, 'no journal unless asked');
|
|
336
|
+
});
|
|
337
|
+
add('write() with release parks, finishes or hands back the instance', async (t) => {
|
|
338
|
+
for (const id of ['parked', 'done', 'failed', 'back']) {
|
|
339
|
+
await t.create(id);
|
|
340
|
+
}
|
|
341
|
+
await t.claim(1, { token: 't' });
|
|
342
|
+
const waits = [{ signal: 'x', key: 'k' }, { signal: 'y', key: null }, { signal: 'a', key: 'z' }];
|
|
343
|
+
equal(await t.store.write('parked', 't', { now: 2, entries: [entry('wait', { kind: 'signal', status: 'pending' })], status: 'suspended', release: { wakeAt: 500, waits, signalCursor: 0 } }), true, 'suspend');
|
|
344
|
+
expect(await t.store.get('parked', { journal: true }), { status: 'suspended', wakeAt: 500, leaseUntil: null, leaseOwner: 'w1', updatedAt: 2, waits, journal: [{ name: 'wait' }] }, 'parked with its waits, in order');
|
|
345
|
+
equal(await t.store.write('parked', 't', t.journalWrite([entry('late')])), false, 'the lease ended with the release');
|
|
346
|
+
const [again] = (await t.claim(500, { token: 't2' })).instances;
|
|
347
|
+
equal(again?.id, 'parked', 'due again at wakeAt');
|
|
348
|
+
await t.store.write('parked', 't2', { now: 501, entries: [], status: 'suspended', release: { wakeAt: 900, waits: [{ signal: 'q', key: 'r' }], signalCursor: 0 } });
|
|
349
|
+
equal((await t.store.get('parked')).waits, [{ signal: 'q', key: 'r' }], 'waits replaced');
|
|
350
|
+
await t.store.write('done', 't', { now: 3, entries: [entry('last')], status: 'completed', output: { ok: [1, 'two'] }, error: null, release: { wakeAt: null, waits: [], signalCursor: 0 } });
|
|
351
|
+
expect(await t.store.get('done'), { status: 'completed', output: { ok: [1, 'two'] }, error: null, wakeAt: null, leaseOwner: 'w1', leaseUntil: null, updatedAt: 3, waits: [] }, 'finished');
|
|
352
|
+
await t.store.write('failed', 't', { now: 4, entries: [], status: 'failed', output: null, error: { name: 'WorkflowFailedError', message: 'no' }, release: { wakeAt: null, waits: [], signalCursor: 0 } });
|
|
353
|
+
expect(await t.store.get('failed'), { status: 'failed', output: null, error: { name: 'WorkflowFailedError', message: 'no' } }, 'a null output stays null');
|
|
354
|
+
await t.store.write('back', 't', { now: 5, entries: [entry('partial', { status: 'pending' })], release: { wakeAt: 5, waits: [], signalCursor: 0 } });
|
|
355
|
+
expect(await t.store.get('back'), { status: 'running', wakeAt: 5, leaseUntil: null, error: null }, 'handed back: due, status unchanged');
|
|
356
|
+
absent((await t.store.get('back')).output, 'output, never set', { orNull: true });
|
|
357
|
+
equal((await t.claim(5, { owner: 'w2', token: 't3' })).instances.map((i) => i.id), ['back'], 'claimable at once');
|
|
358
|
+
});
|
|
359
|
+
add('write() sets the custom status under the lease, and leaves it as it is when not given', async (t) => {
|
|
360
|
+
await t.create('a');
|
|
361
|
+
await t.claim(1, { token: 't' });
|
|
362
|
+
const status = { stage: 'packing', items: [{ sku: 'kibble-2kg', qty: 2 }], note: "it's 100% ü", none: null };
|
|
363
|
+
equal(await t.store.write('a', 't', { now: 2, entries: [], customStatus: status }), true, 'set');
|
|
364
|
+
expect(await t.store.get('a'), { customStatus: status, updatedAt: 2 }, 'the status');
|
|
365
|
+
await t.store.write('a', 't', { now: 3, entries: [entry('x')], status: 'running' });
|
|
366
|
+
expect(await t.store.get('a'), { customStatus: status }, 'a write without it leaves it');
|
|
367
|
+
await t.store.write('a', 't', { now: 4, entries: [], customStatus: 'shipped' });
|
|
368
|
+
expect((await t.store.list({ limit: 10, offset: 0 }))[0], { customStatus: 'shipped' }, 'a string, in list()');
|
|
369
|
+
await t.store.write('a', 't', { now: 5, entries: [], customStatus: 0 });
|
|
370
|
+
equal((await t.store.get('a')).customStatus, 0, 'a falsy value');
|
|
371
|
+
await t.store.write('a', 't', { now: 6, entries: [], customStatus: null, release: { wakeAt: 50, waits: [], signalCursor: 0 } });
|
|
372
|
+
equal((await t.store.get('a')).customStatus, null, 'cleared, with a release');
|
|
373
|
+
equal(await t.store.write('a', 't', { now: 7, entries: [], customStatus: 'stale' }), false, 'after the release');
|
|
374
|
+
equal((await t.store.get('a')).customStatus, null, 'not written under a stale token');
|
|
375
|
+
});
|
|
376
|
+
// ---------------------------------------------------------------- signals
|
|
377
|
+
add('signal() matches keys exactly (null, empty and other keys all differ) and wakes suspended instances only', async (t) => {
|
|
378
|
+
for (const id of ['none', 'empty', 'k1', 'running']) {
|
|
379
|
+
await t.create(id);
|
|
380
|
+
}
|
|
381
|
+
await t.claim(1, { token: 't' });
|
|
382
|
+
const keys = { none: null, empty: '', k1: 'k1' };
|
|
383
|
+
for (const id of Object.keys(keys)) {
|
|
384
|
+
await t.store.write(id, 't', { now: 2, entries: [], status: 'suspended', release: { wakeAt: null, waits: [{ signal: 'shipment.delivered', key: keys[id] }], signalCursor: 0 } });
|
|
385
|
+
}
|
|
386
|
+
const s1 = await t.store.signal({ name: 'shipment.delivered', key: 'k2', dedupeId: null, payload: 'x', now: 3 });
|
|
387
|
+
equal(s1.woken, 0, 'another key');
|
|
388
|
+
const s2 = await t.store.signal({ name: 'other', key: null, dedupeId: null, payload: 'x', now: 3 });
|
|
389
|
+
equal(s2.woken, 0, 'another name');
|
|
390
|
+
const s3 = await t.store.signal({ name: 'shipment.delivered', key: '', dedupeId: null, payload: { e: '' }, now: 4 });
|
|
391
|
+
equal(s3.woken, 1, "the '' key");
|
|
392
|
+
expect(await t.store.get('empty'), { wakeAt: 4, updatedAt: 4 }, "woken: the '' wait");
|
|
393
|
+
expect(await t.store.get('none'), { wakeAt: null }, 'not woken: the wait without a key');
|
|
394
|
+
const s4 = await t.store.signal({ name: 'shipment.delivered', key: null, dedupeId: null, payload: null, now: 5 });
|
|
395
|
+
equal(s4.woken, 1, 'no key');
|
|
396
|
+
const s5 = await t.store.signal({ name: 'shipment.delivered', key: 'k1', dedupeId: null, payload: [1], now: 6 });
|
|
397
|
+
const s6 = await t.store.signal({ name: 'shipment.delivered', key: 'k1', dedupeId: null, payload: [2], now: 7 });
|
|
398
|
+
equal([s5.woken, s6.woken], [1, 0], 'an instance already due is not woken again');
|
|
399
|
+
const ids = [s1, s2, s3, s4, s5, s6].map((s) => s.id);
|
|
400
|
+
equal([...ids].sort((a, b) => a - b), ids, 'ids increase');
|
|
401
|
+
equal(new Set(ids).size, 6, 'ids are unique');
|
|
402
|
+
const read = (key, afterId = 0, upToId = s6.id) => t.store.signals({ name: 'shipment.delivered', key, afterId, upToId });
|
|
403
|
+
equal(await read(null), [{ id: s4.id, name: 'shipment.delivered', key: null, payload: null, createdAt: 5 }], 'signals() without a key');
|
|
404
|
+
equal(await read(''), [{ id: s3.id, name: 'shipment.delivered', key: '', payload: { e: '' }, createdAt: 4 }], "signals() with ''");
|
|
405
|
+
equal((await read('k1')).map((s) => s.id), [s5.id, s6.id], 'signals() by id');
|
|
406
|
+
equal((await read('k1', s5.id)).map((s) => s.id), [s6.id], 'afterId is exclusive');
|
|
407
|
+
equal((await read('k1', 0, s5.id)).map((s) => s.id), [s5.id], 'upToId is inclusive');
|
|
408
|
+
equal((await t.claim(0)).lastSignalId, s6.id, 'the last signal id');
|
|
409
|
+
expect(await t.store.get('running'), { status: 'running' }, 'a running instance is left to write()');
|
|
410
|
+
});
|
|
411
|
+
add('write() keeps an instance due when a signal for its new waits arrived after its cursor', async (t) => {
|
|
412
|
+
await t.create('a');
|
|
413
|
+
await t.create('b');
|
|
414
|
+
const { lastSignalId: cursor } = await t.claim(1, { token: 't' });
|
|
415
|
+
await t.store.signal({ name: 'go', key: 'a', dedupeId: null, payload: 1, now: 2 }); // while a and b were executing
|
|
416
|
+
const suspend = (id) => t.store.write(id, 't', { now: 3, entries: [], status: 'suspended', release: { wakeAt: FAR, waits: [{ signal: 'go', key: id }], signalCursor: cursor } });
|
|
417
|
+
equal(await suspend('a'), true, 'suspend a');
|
|
418
|
+
equal(await suspend('b'), true, 'suspend b');
|
|
419
|
+
expect(await t.store.get('a'), { status: 'suspended', wakeAt: 3, leaseUntil: null }, 'a saw the signal: due now');
|
|
420
|
+
expect(await t.store.get('b'), { status: 'suspended', wakeAt: FAR, waits: [{ signal: 'go', key: 'b' }] }, 'b: another key, parked');
|
|
421
|
+
const [c] = (await t.claim(3, { token: 't2' })).instances;
|
|
422
|
+
equal(c?.id, 'a', 'a is claimable');
|
|
423
|
+
await t.store.write('a', 't2', { now: 4, entries: [], status: 'suspended', release: { wakeAt: FAR, waits: [{ signal: 'go', key: 'a' }], signalCursor: (await t.claim(0)).lastSignalId } });
|
|
424
|
+
expect(await t.store.get('a'), { wakeAt: FAR }, 'a signal at or below the cursor was seen: parked');
|
|
425
|
+
});
|
|
426
|
+
add('write() with a signal records it as signal() does, only when the write lands', async (t) => {
|
|
427
|
+
await t.create('parent', 0);
|
|
428
|
+
await t.create('child', 1);
|
|
429
|
+
await t.claim(1, { token: 'p', limit: 1 });
|
|
430
|
+
const parked = { wakeAt: FAR, waits: [{ signal: 'child.ended', key: 'child' }], signalCursor: 0 };
|
|
431
|
+
await t.store.write('parent', 'p', { now: 2, entries: [], status: 'suspended', release: parked });
|
|
432
|
+
await t.claim(3, { token: 'stale', leaseUntil: 4 });
|
|
433
|
+
await t.claim(5, { token: 'c' });
|
|
434
|
+
const ended = { name: 'child.ended', key: 'child', dedupeId: 'child', payload: { status: 'completed', output: { label: 'LBL-1' } }, now: 6 };
|
|
435
|
+
const finish = (token, now) => t.store.write('child', token, { now, entries: [], status: 'completed', output: 1, error: null, release: { wakeAt: null, waits: [], signalCursor: 0 }, signal: { ...ended, now } });
|
|
436
|
+
equal(await finish('stale', 6), false, 'a stale write');
|
|
437
|
+
equal((await t.claim(0)).lastSignalId, 0, 'no signal from the stale write');
|
|
438
|
+
expect(await t.store.get('parent'), { wakeAt: FAR }, 'nothing woken');
|
|
439
|
+
equal(await finish('c', 7), true, 'the lease holder ends the child');
|
|
440
|
+
const { lastSignalId } = await t.claim(0);
|
|
441
|
+
equal(await t.store.signals({ name: 'child.ended', key: 'child', afterId: 0, upToId: lastSignalId }), [{ id: lastSignalId, name: 'child.ended', key: 'child', payload: ended.payload, createdAt: 7 }], 'the signal, recorded');
|
|
442
|
+
expect(await t.store.get('parent'), { status: 'suspended', wakeAt: 7, updatedAt: 7 }, 'the parent, woken');
|
|
443
|
+
expect(await t.store.get('child'), { status: 'completed', output: 1 }, 'the child, ended');
|
|
444
|
+
// Deduplicated as signal() does: a retried child that ends again stores nothing more.
|
|
445
|
+
await t.store.signal({ name: 'other', key: null, dedupeId: null, payload: null, now: 8 });
|
|
446
|
+
await t.claim(0, { token: 'again' }); // nothing is due; a claim of a finished child never happens
|
|
447
|
+
expect(await t.store.signal({ ...ended, now: 9 }), { id: lastSignalId, created: false, woken: 0 }, 'the same dedupe id');
|
|
448
|
+
});
|
|
449
|
+
add('signal() with a dedupeId stores the signal once per name, and a repeat writes and wakes nothing', async (t) => {
|
|
450
|
+
await t.create('a');
|
|
451
|
+
await t.claim(1, { token: 't' });
|
|
452
|
+
const waits = [{ signal: 'payment.captured', key: 'o1' }];
|
|
453
|
+
await t.store.write('a', 't', { now: 1, entries: [], status: 'suspended', release: { wakeAt: FAR, waits, signalCursor: 0 } });
|
|
454
|
+
const send = (name, key, dedupeId, payload, now) => t.store.signal({ name, key, dedupeId, payload, now });
|
|
455
|
+
const first = await send('payment.captured', 'o1', 'ch_1', { amount: 2499 }, 2);
|
|
456
|
+
expect(first, { created: true, woken: 1, key: 'o1' }, 'the first signal');
|
|
457
|
+
// Parked again on the same wait, having seen the first signal.
|
|
458
|
+
await t.claim(2, { token: 't2' });
|
|
459
|
+
await t.store.write('a', 't2', { now: 3, entries: [], status: 'suspended', release: { wakeAt: FAR, waits, signalCursor: first.id } });
|
|
460
|
+
equal(await send('payment.captured', 'o1', 'ch_1', { amount: 1 }, 4), { id: first.id, woken: 0, created: false, key: 'o1' }, 'a repeat');
|
|
461
|
+
expect(await t.store.get('a'), { status: 'suspended', wakeAt: FAR, updatedAt: 3 }, 'not woken by the repeat');
|
|
462
|
+
equal(await send('payment.captured', 'o2', 'ch_1', 'x', 5), { id: first.id, woken: 0, created: false, key: 'o1' }, 'the same id with another key');
|
|
463
|
+
const otherName = await send('refund.issued', 'o1', 'ch_1', 'y', 6);
|
|
464
|
+
expect(otherName, { created: true, key: 'o1' }, 'the same id under another name');
|
|
465
|
+
const plain = [await send('payment.captured', 'o1', null, 'p', 7), await send('payment.captured', 'o1', null, 'p', 8)];
|
|
466
|
+
expect(plain, [{ created: true }, { created: true }], 'signals without a dedupeId');
|
|
467
|
+
equal(new Set([first.id, otherName.id, ...plain.map((s) => s.id)]).size, 4, 'distinct ids');
|
|
468
|
+
const read = (name, key) => t.store.signals({ name, key, afterId: 0, upToId: Number.MAX_SAFE_INTEGER });
|
|
469
|
+
equal((await read('payment.captured', 'o1')).map((s) => s.payload), [{ amount: 2499 }, 'p', 'p'], 'the first payload, stored once');
|
|
470
|
+
equal(await read('payment.captured', 'o2'), [], 'nothing under the other key');
|
|
471
|
+
equal((await t.claim(0)).lastSignalId, plain[1].id, 'the last signal id');
|
|
472
|
+
});
|
|
473
|
+
// ---------------------------------------------------------------- retention
|
|
474
|
+
add('purge() deletes old finished instances of the given statuses, oldest first, with their journals', async (t) => {
|
|
475
|
+
const finish = async (id, status, at) => {
|
|
476
|
+
await t.create(id);
|
|
477
|
+
await t.claim(0, { token: `t-${id}`, leaseUntil: FAR });
|
|
478
|
+
await t.store.write(id, `t-${id}`, { now: at, entries: [entry('only')], status, error: null, release: { wakeAt: null, waits: [], signalCursor: 0 } });
|
|
479
|
+
};
|
|
480
|
+
await finish('done-2', 'completed', 200);
|
|
481
|
+
await finish('done-1', 'completed', 100);
|
|
482
|
+
await finish('failed', 'failed', 150);
|
|
483
|
+
await finish('stuck', 'compensation_failed', 50);
|
|
484
|
+
await finish('recent', 'completed', 1_000);
|
|
485
|
+
await t.create('pending');
|
|
486
|
+
await t.create('parked');
|
|
487
|
+
await t.claim(0, { token: 'p', leaseUntil: FAR, limit: 10 });
|
|
488
|
+
await t.store.write('parked', 'p', { now: 10, entries: [entry('wait', { kind: 'signal', status: 'pending' })], status: 'suspended', release: { wakeAt: null, waits: [{ signal: 's', key: 'k' }], signalCursor: 0 } });
|
|
489
|
+
const statuses = ['completed', 'failed', 'cancelled'];
|
|
490
|
+
equal(await t.store.purge({ statuses: [...statuses], before: 500, limit: 2 }), { instances: 2, signals: 0, rateLimits: 0 }, 'the first batch');
|
|
491
|
+
equal(await t.store.get('done-1'), null, 'the oldest, gone');
|
|
492
|
+
equal(await t.store.get('failed'), null, 'the next oldest, gone');
|
|
493
|
+
expect(await t.store.get('done-2', { journal: true }), { status: 'completed', journal: [{ name: 'only' }] }, 'past the limit: kept');
|
|
494
|
+
equal(await t.store.purge({ statuses: [...statuses], before: 500, limit: 2 }), { instances: 1, signals: 0, rateLimits: 0 }, 'the next batch');
|
|
495
|
+
equal(await t.store.purge({ statuses: [...statuses], before: 500, limit: 2 }), { instances: 0, signals: 0, rateLimits: 0 }, 'nothing left');
|
|
496
|
+
const left = (await t.store.list({ limit: 100, offset: 0 })).map((i) => i.id).sort();
|
|
497
|
+
equal(left, ['parked', 'pending', 'recent', 'stuck'], 'unfinished, too recent, and another status: kept');
|
|
498
|
+
expect(await t.store.get('parked', { journal: true }), { waits: [{ signal: 's', key: 'k' }], journal: [{ name: 'wait' }] }, 'an unfinished instance, untouched');
|
|
499
|
+
equal(await t.store.purge({ statuses: ['compensation_failed'], before: 500, limit: 10 }), { instances: 1, signals: 0, rateLimits: 0 }, 'compensation_failed, when asked for');
|
|
500
|
+
// A purged id can be started again.
|
|
501
|
+
expect(await t.create('done-1'), { created: true, instance: { status: 'pending' } }, 'the id, free again');
|
|
502
|
+
});
|
|
503
|
+
add('purge() deletes signals no instance can take any more, and never the newest', async (t) => {
|
|
504
|
+
const send = (payload, now, dedupeId = null) => t.store.signal({ name: 's', key: 'k', dedupeId, payload, now });
|
|
505
|
+
await send(1, 10, 'evt-1');
|
|
506
|
+
const s2 = await send(2, 20);
|
|
507
|
+
await t.create('waiting'); // takes signals above s2 only
|
|
508
|
+
const s3 = await send(3, 30);
|
|
509
|
+
const s4 = await send(4, 40);
|
|
510
|
+
const all = () => t.store.signals({ name: 's', key: 'k', afterId: 0, upToId: FAR });
|
|
511
|
+
equal(await t.store.purge({ statuses: ['completed'], before: 1_000, limit: 1 }), { instances: 0, signals: 1, rateLimits: 0 }, 'one at a time');
|
|
512
|
+
equal((await all()).map((s) => s.id), [s2.id, s3.id, s4.id], 'the lowest id first');
|
|
513
|
+
equal(await t.store.purge({ statuses: ['completed'], before: 1_000, limit: 10 }), { instances: 0, signals: 1, rateLimits: 0 }, 'up to the cursor');
|
|
514
|
+
equal((await all()).map((s) => s.id), [s3.id, s4.id], "the signals an unfinished instance can still take: kept");
|
|
515
|
+
await t.claim(0, { token: 't', leaseUntil: FAR });
|
|
516
|
+
await t.store.write('waiting', 't', { now: 50, entries: [], status: 'completed', error: null, release: { wakeAt: null, waits: [], signalCursor: s4.id } });
|
|
517
|
+
equal(await t.store.purge({ statuses: ['completed'], before: 35, limit: 10 }), { instances: 0, signals: 1, rateLimits: 0 }, 'only the old enough');
|
|
518
|
+
equal((await all()).map((s) => s.id), [s4.id], 's3 is old enough, s4 too recent');
|
|
519
|
+
equal(await t.store.purge({ statuses: ['completed'], before: 1_000, limit: 10 }), { instances: 1, signals: 0, rateLimits: 0 }, 'the newest stays');
|
|
520
|
+
equal((await t.claim(0)).lastSignalId, s4.id, 'the last signal id');
|
|
521
|
+
expect(await t.create('later'), { instance: { signalCursor: s4.id } }, "a new instance's cursor");
|
|
522
|
+
expect(await send(5, 60), { created: true }, 'the next signal');
|
|
523
|
+
expect(await send(1, 70, 'evt-1'), { created: true }, 'a dedupe id, stored again once its signal was purged');
|
|
524
|
+
});
|
|
525
|
+
add('purge() deletes the rate-limit windows that ended before `before`, oldest first, and keeps the open ones', async (t) => {
|
|
526
|
+
for (const key of ['a', 'b', 'c']) {
|
|
527
|
+
await t.prioritized(key, 0, { rateLimitKey: key });
|
|
528
|
+
}
|
|
529
|
+
const rateLimits = (duration) => [{ workflow: W.name, limit: null, perKey: { max: 1, duration } }];
|
|
530
|
+
await t.claim(10, { rateLimits: rateLimits(100), limit: 1, leaseUntil: 11 }); // a: window ends at 110
|
|
531
|
+
await t.claim(20, { rateLimits: rateLimits(50), limit: 1, leaseUntil: 21 }); // a is full; b: ends at 70
|
|
532
|
+
await t.claim(30, { rateLimits: rateLimits(5_000), limit: 1, leaseUntil: 31 }); // c: ends at 5030
|
|
533
|
+
equal(await t.store.purge({ statuses: ['completed'], before: 200, limit: 1 }), { instances: 0, signals: 0, rateLimits: 1 }, "b's, which ended first");
|
|
534
|
+
equal(await t.store.purge({ statuses: ['completed'], before: 200, limit: 10 }), { instances: 0, signals: 0, rateLimits: 1 }, "a's");
|
|
535
|
+
equal(await t.store.purge({ statuses: ['completed'], before: 200, limit: 10 }), { instances: 0, signals: 0, rateLimits: 0 }, "c's is open");
|
|
536
|
+
equal((await t.claim(40, { rateLimits: rateLimits(5_000), leaseUntil: 41 })).instances.map((i) => i.id), ['a', 'b'], 'the purged windows, as good as new');
|
|
537
|
+
});
|
|
538
|
+
// ---------------------------------------------------------------- cancel
|
|
539
|
+
add('requestCancel() accepts once, for pending, running and suspended instances only', async (t) => {
|
|
540
|
+
for (const id of ['pending', 'running', 'suspended', 'compensating', 'done']) {
|
|
541
|
+
await t.create(id, 0);
|
|
542
|
+
}
|
|
543
|
+
await t.create('pending-later', 50);
|
|
544
|
+
await t.claim(1, { token: 't' });
|
|
545
|
+
const park = { now: 1, entries: [], release: { wakeAt: FAR, waits: [], signalCursor: 0 } };
|
|
546
|
+
await t.store.write('pending', 't', { ...park, status: 'pending' });
|
|
547
|
+
await t.store.write('suspended', 't', { ...park, status: 'suspended' });
|
|
548
|
+
await t.store.write('compensating', 't', { now: 1, entries: [], status: 'compensating', error: { name: 'E', message: 'x' } });
|
|
549
|
+
await t.store.write('done', 't', { now: 1, entries: [], status: 'completed', error: null, release: { wakeAt: null, waits: [], signalCursor: 0 } });
|
|
550
|
+
equal(await t.store.requestCancel('suspended', { reason: 'Changed my mind.', now: 10, terminate: false }), true, 'suspended');
|
|
551
|
+
expect(await t.store.get('suspended'), { status: 'suspended', cancelRequested: true, cancelReason: 'Changed my mind.', wakeAt: 10, updatedAt: 10 }, 'due now');
|
|
552
|
+
equal(await t.store.requestCancel('suspended', { reason: 'Again.', now: 11, terminate: false }), false, 'a repeated request');
|
|
553
|
+
expect(await t.store.get('suspended'), { cancelReason: 'Changed my mind.', wakeAt: 10 }, 'the first reason stays');
|
|
554
|
+
equal(await t.store.requestCancel('pending', { reason: null, now: 12, terminate: false }), true, 'pending');
|
|
555
|
+
expect(await t.store.get('pending'), { cancelRequested: true, cancelReason: null, wakeAt: 12 }, 'pending, due now');
|
|
556
|
+
equal(await t.store.requestCancel('pending-later', { reason: null, now: 12, terminate: false }), true, 'pending, not yet due');
|
|
557
|
+
expect(await t.store.get('pending-later'), { wakeAt: 12 }, 'due now');
|
|
558
|
+
equal(await t.store.requestCancel('running', { reason: 'r', now: 13, terminate: false }), true, 'running');
|
|
559
|
+
expect(await t.store.get('running'), { cancelRequested: true, wakeAt: 0 }, 'an instance already due keeps its wakeAt');
|
|
560
|
+
equal(await t.store.requestCancel('compensating', { reason: 'x', now: 14, terminate: false }), false, 'compensating');
|
|
561
|
+
equal(await t.store.requestCancel('done', { reason: 'x', now: 15, terminate: false }), false, 'finished');
|
|
562
|
+
equal(await t.store.requestCancel('missing', { reason: 'x', now: 16, terminate: false }), false, 'unknown');
|
|
563
|
+
expect(await t.store.get('done'), { status: 'completed', cancelRequested: false }, 'finished: unchanged');
|
|
564
|
+
});
|
|
565
|
+
add('requestCancel() with terminate accepts once, also after a cancel and for a compensating instance', async (t) => {
|
|
566
|
+
for (const id of ['suspended', 'cancelled-first', 'compensating', 'done']) {
|
|
567
|
+
await t.create(id, 0);
|
|
568
|
+
}
|
|
569
|
+
await t.claim(1, { token: 't' });
|
|
570
|
+
await t.store.write('suspended', 't', { now: 1, entries: [], status: 'suspended', release: { wakeAt: FAR, waits: [], signalCursor: 0 } });
|
|
571
|
+
await t.store.write('compensating', 't', { now: 1, entries: [], status: 'compensating', error: { name: 'E', message: 'x' }, release: { wakeAt: FAR, waits: [], signalCursor: 0 } });
|
|
572
|
+
await t.store.write('done', 't', { now: 1, entries: [], status: 'completed', error: null, release: { wakeAt: null, waits: [], signalCursor: 0 } });
|
|
573
|
+
const terminate = (id, reason, now) => t.store.requestCancel(id, { reason, now, terminate: true });
|
|
574
|
+
equal(await terminate('suspended', 'Stuck on a retired carrier.', 10), true, 'suspended');
|
|
575
|
+
expect(await t.store.get('suspended'), { status: 'suspended', cancelRequested: true, terminateRequested: true, cancelReason: 'Stuck on a retired carrier.', wakeAt: 10, updatedAt: 10 }, 'terminate requested, due now');
|
|
576
|
+
equal(await terminate('suspended', 'Again.', 11), false, 'a repeated terminate');
|
|
577
|
+
equal(await t.store.requestCancel('suspended', { reason: 'Later.', now: 12, terminate: false }), false, 'a cancel after it');
|
|
578
|
+
expect(await t.store.get('suspended'), { cancelReason: 'Stuck on a retired carrier.' }, 'the reason stays');
|
|
579
|
+
equal(await t.store.requestCancel('cancelled-first', { reason: 'Cancel.', now: 13, terminate: false }), true, 'a cancel first');
|
|
580
|
+
expect(await t.store.get('cancelled-first'), { cancelRequested: true, terminateRequested: false }, 'cancelled, not terminated');
|
|
581
|
+
equal(await terminate('cancelled-first', 'Terminate.', 14), true, 'a terminate after a cancel');
|
|
582
|
+
expect(await t.store.get('cancelled-first'), { status: 'running', terminateRequested: true, cancelReason: 'Terminate.' }, 'its reason replaced');
|
|
583
|
+
equal(await terminate('compensating', 'Stop undoing.', 15), true, 'compensating');
|
|
584
|
+
expect(await t.store.get('compensating'), { status: 'compensating', cancelRequested: true, terminateRequested: true, wakeAt: 15 }, 'due now');
|
|
585
|
+
equal(await terminate('done', 'x', 16), false, 'finished');
|
|
586
|
+
equal(await terminate('missing', 'x', 17), false, 'unknown');
|
|
587
|
+
expect(await t.store.get('done'), { status: 'completed', cancelRequested: false, terminateRequested: false }, 'finished: unchanged');
|
|
588
|
+
});
|
|
589
|
+
add('write() keeps a suspending instance with a pending cancel due, but not a compensating one', async (t) => {
|
|
590
|
+
await t.create('a');
|
|
591
|
+
await t.claim(1, { token: 't' });
|
|
592
|
+
await t.store.requestCancel('a', { reason: null, now: 2, terminate: false });
|
|
593
|
+
await t.store.write('a', 't', { now: 20, entries: [], status: 'suspended', release: { wakeAt: FAR, waits: [], signalCursor: 0 } });
|
|
594
|
+
expect(await t.store.get('a'), { status: 'suspended', wakeAt: 20 }, 'suspended with a cancel: due now');
|
|
595
|
+
await t.claim(20, { token: 't2' });
|
|
596
|
+
await t.store.write('a', 't2', { now: 21, entries: [], status: 'compensating', error: { name: 'WorkflowCancelledError', message: 'Cancelled.' }, release: { wakeAt: 500, waits: [], signalCursor: 0 } });
|
|
597
|
+
expect(await t.store.get('a'), { status: 'compensating', wakeAt: 500 }, 'compensating: its own wakeAt');
|
|
598
|
+
});
|
|
599
|
+
// ---------------------------------------------------------------- operator actions
|
|
600
|
+
add('reopen() applies once, only while the instance is unleased and as the engine read it', async (t) => {
|
|
601
|
+
await t.create('a', 0);
|
|
602
|
+
await t.claim(1, { token: 't' });
|
|
603
|
+
const failure = { name: 'StepFailedError', message: 'no' };
|
|
604
|
+
await t.store.write('a', 't', { now: 2, entries: [entry('charge', { status: 'failed', attempts: 3, error: failure })], status: 'failed', error: failure, release: { wakeAt: null, waits: [], signalCursor: 0 } });
|
|
605
|
+
const reopen = (expect, now) => t.store.reopen('a', {
|
|
606
|
+
expect,
|
|
607
|
+
status: 'pending',
|
|
608
|
+
error: null,
|
|
609
|
+
deadline: 9_000,
|
|
610
|
+
entries: [entry('charge', { status: 'pending', attempts: 0, wakeAt: null, error: failure }), entry('$retry:1', { kind: 'retry', attempts: 0, data: { from: 'failed' } })],
|
|
611
|
+
now,
|
|
612
|
+
});
|
|
613
|
+
equal(await reopen({ status: 'completed', runs: 1 }, 3), false, 'another status');
|
|
614
|
+
equal(await reopen({ status: 'failed', runs: 2 }, 3), false, 'another runs count');
|
|
615
|
+
expect(await t.store.get('a', { journal: true }), { status: 'failed', error: failure, updatedAt: 2, deadline: null, journal: [{ name: 'charge', status: 'failed' }] }, 'unchanged');
|
|
616
|
+
equal(await reopen({ status: 'failed', runs: 1 }, 5), true, 'as read');
|
|
617
|
+
expect(await t.store.get('a', { journal: true }), {
|
|
618
|
+
status: 'pending',
|
|
619
|
+
error: null,
|
|
620
|
+
wakeAt: 5,
|
|
621
|
+
updatedAt: 5,
|
|
622
|
+
deadline: 9_000,
|
|
623
|
+
journal: [{ name: 'charge', status: 'pending', attempts: 0 }, { name: '$retry:1', kind: 'retry', data: { from: 'failed' } }],
|
|
624
|
+
}, 'reopened');
|
|
625
|
+
equal(await reopen({ status: 'failed', runs: 1 }, 6), false, 'a second retry of the same read');
|
|
626
|
+
const [claimed] = (await t.claim(5, { token: 't2' })).instances;
|
|
627
|
+
expect(claimed, { id: 'a', status: 'running', runs: 2 }, 'claimable');
|
|
628
|
+
await t.store.write('a', 't2', { now: 7, entries: [], status: 'compensation_failed', error: failure });
|
|
629
|
+
equal(await t.store.reopen('a', { expect: { status: 'compensation_failed', runs: 2 }, status: 'compensating', error: failure, entries: [], now: 8 }), false, 'leased');
|
|
630
|
+
await t.store.write('a', 't2', { now: 9, entries: [], release: { wakeAt: null, waits: [], signalCursor: 0 } });
|
|
631
|
+
equal(await t.store.reopen('a', { expect: { status: 'compensation_failed', runs: 2 }, status: 'compensating', error: failure, entries: [], now: 10 }), true, 'unleased');
|
|
632
|
+
expect(await t.store.get('a'), { status: 'compensating', wakeAt: 10, deadline: 9_000 }, 'the deadline, left as it was');
|
|
633
|
+
});
|
|
634
|
+
add('delete() removes an instance with its journal and waits, only in the given statuses', async (t) => {
|
|
635
|
+
await t.create('done');
|
|
636
|
+
await t.create('parked');
|
|
637
|
+
await t.claim(1, { token: 't' });
|
|
638
|
+
await t.store.write('done', 't', { now: 2, entries: [entry('x')], status: 'completed', error: null, release: { wakeAt: null, waits: [], signalCursor: 0 } });
|
|
639
|
+
await t.store.write('parked', 't', { now: 2, entries: [entry('w', { kind: 'signal', status: 'pending' })], status: 'suspended', release: { wakeAt: null, waits: [{ signal: 's', key: 'k' }], signalCursor: 0 } });
|
|
640
|
+
equal(await t.store.delete('parked', ['completed', 'failed']), false, 'another status');
|
|
641
|
+
equal(await t.store.delete('done', ['completed', 'failed']), true, 'deleted');
|
|
642
|
+
equal(await t.store.get('done'), null, 'gone');
|
|
643
|
+
equal(await t.store.delete('done', ['completed']), false, 'already gone');
|
|
644
|
+
equal(await t.store.delete('missing', ['completed']), false, 'unknown');
|
|
645
|
+
await t.claim(3, { token: 't2' }); // not due: nothing claimed
|
|
646
|
+
equal(await t.store.delete('parked', ['suspended']), true, 'an unfinished one, when asked');
|
|
647
|
+
equal(await t.store.list({ limit: 10, offset: 0 }), [], 'none left');
|
|
648
|
+
expect(await t.create('done'), { created: true }, 'the id, free again');
|
|
649
|
+
expect(await t.store.signal({ name: 's', key: 'k', dedupeId: null, payload: 1, now: 4 }), { woken: 0 }, "the deleted instance's waits are gone");
|
|
650
|
+
await t.claim(5, { token: 't3' });
|
|
651
|
+
equal(await t.store.delete('done', ['running']), true, 'a leased one');
|
|
652
|
+
equal(await t.store.write('done', 't3', t.journalWrite([entry('late')])), false, "its worker's next write");
|
|
653
|
+
equal(await t.store.renew('done', 't3', 100), null, 'and renewal');
|
|
654
|
+
});
|
|
655
|
+
// ---------------------------------------------------------------- schedules (the core's ScheduleStore contract)
|
|
656
|
+
scheduleCases((name, run) => add(name, (t) => run(workflowScheduleStore(() => t.store))), 'base');
|
|
657
|
+
// ---------------------------------------------------------------- the application's transaction
|
|
658
|
+
if (transaction) {
|
|
659
|
+
add("createInTransaction() and signalInTransaction() commit and roll back with the application's transaction", async (t) => {
|
|
660
|
+
const { createInTransaction, signalInTransaction } = requireTransactionMethods(t.store);
|
|
661
|
+
await rejects(transaction(async (tx) => {
|
|
662
|
+
expect(await createInTransaction(tx, { id: 'rolled-back', workflow: 'order-fulfilment', version: 1, input: 1, deadline: null, now: 1 }), { created: true }, 'created in the transaction');
|
|
663
|
+
await signalInTransaction(tx, { name: 'go', key: 'x', dedupeId: null, payload: 1, now: 1 });
|
|
664
|
+
throw new Error('payment declined');
|
|
665
|
+
}), 'payment declined');
|
|
666
|
+
equal(await t.store.get('rolled-back'), null, 'rolled back: no instance');
|
|
667
|
+
equal((await t.claim(0)).lastSignalId, 0, 'rolled back: no signal');
|
|
668
|
+
await t.create('waiting');
|
|
669
|
+
await t.claim(1, { token: 't' });
|
|
670
|
+
await t.store.write('waiting', 't', { now: 1, entries: [], status: 'suspended', release: { wakeAt: null, waits: [{ signal: 'go', key: 'w' }], signalCursor: 0 } });
|
|
671
|
+
const results = await transaction(async (tx) => [
|
|
672
|
+
await createInTransaction(tx, { id: 'committed', workflow: 'order-fulfilment', version: 1, input: { a: 1 }, deadline: 5_000, now: 2 }),
|
|
673
|
+
await createInTransaction(tx, { id: 'committed', workflow: 'order-fulfilment', version: 1, input: { a: 2 }, deadline: null, now: 3 }),
|
|
674
|
+
await signalInTransaction(tx, { name: 'go', key: 'w', dedupeId: null, payload: 'p', now: 4 }),
|
|
675
|
+
]);
|
|
676
|
+
expect(results[0], { created: true, instance: { id: 'committed', status: 'pending', input: { a: 1 }, signalCursor: 0 } }, 'created');
|
|
677
|
+
expect(results[1], { created: false, instance: { id: 'committed', input: { a: 1 } } }, 'the row written earlier in the transaction');
|
|
678
|
+
equal(results[2].woken, 1, 'woken');
|
|
679
|
+
expect(await t.store.get('committed'), { status: 'pending', wakeAt: 2 }, 'committed');
|
|
680
|
+
expect(await t.store.get('waiting'), { wakeAt: 4 }, 'woken with the commit');
|
|
681
|
+
expect(await t.store.signals({ name: 'go', key: 'w', afterId: 0, upToId: results[2].id }), [{ id: results[2].id, payload: 'p' }], 'the signal');
|
|
682
|
+
});
|
|
683
|
+
}
|
|
684
|
+
if (transaction) {
|
|
685
|
+
add('signalInTransaction() stores a dedupeId once per commit, and a rolled-back signal leaves its id free', async (t) => {
|
|
686
|
+
const { signalInTransaction } = requireTransactionMethods(t.store);
|
|
687
|
+
const data = (payload, now) => ({ name: 'payment.captured', key: 'o1', dedupeId: 'ch_1', payload, now });
|
|
688
|
+
await rejects(transaction(async (tx) => {
|
|
689
|
+
expect(await signalInTransaction(tx, data('rolled back', 1)), { created: true }, 'stored in the transaction');
|
|
690
|
+
throw new Error('payment declined');
|
|
691
|
+
}), 'payment declined');
|
|
692
|
+
const [first, again] = await transaction(async (tx) => [await signalInTransaction(tx, data('committed', 2)), await signalInTransaction(tx, data('again', 3))]);
|
|
693
|
+
expect(first, { created: true }, 'the id is free after the rollback');
|
|
694
|
+
equal(again, { id: first.id, woken: 0, created: false, key: 'o1' }, 'a repeat in the same transaction');
|
|
695
|
+
expect(await t.store.signal(data('outside', 4)), { id: first.id, created: false }, 'a repeat after the commit');
|
|
696
|
+
equal((await t.store.signals({ name: 'payment.captured', key: 'o1', afterId: 0, upToId: Number.MAX_SAFE_INTEGER })).map((s) => s.payload), ['committed'], 'one signal');
|
|
697
|
+
});
|
|
698
|
+
}
|
|
699
|
+
// ---------------------------------------------------------------- concurrency
|
|
700
|
+
if (options.concurrent) {
|
|
701
|
+
add('concurrent claims never return the same instance twice', async (t) => {
|
|
702
|
+
for (let i = 0; i < 60; i++) {
|
|
703
|
+
await t.create(`i${String(i).padStart(2, '0')}`, i);
|
|
704
|
+
}
|
|
705
|
+
const owners = new Map();
|
|
706
|
+
const claimer = async (owner) => {
|
|
707
|
+
for (;;) {
|
|
708
|
+
const { instances } = await t.claim(1_000, { owner, token: randomUUID(), limit: 7, leaseUntil: FAR });
|
|
709
|
+
if (instances.length === 0) {
|
|
710
|
+
return;
|
|
711
|
+
}
|
|
712
|
+
for (const instance of instances) {
|
|
713
|
+
if (owners.has(instance.id)) {
|
|
714
|
+
throw new Error(`${instance.id} was claimed by ${owners.get(instance.id)} and ${owner}`);
|
|
715
|
+
}
|
|
716
|
+
owners.set(instance.id, owner);
|
|
717
|
+
}
|
|
718
|
+
await jitter();
|
|
719
|
+
}
|
|
720
|
+
};
|
|
721
|
+
await Promise.all(['c1', 'c2', 'c3', 'c4', 'c5'].map(claimer));
|
|
722
|
+
equal(owners.size, 60, 'every instance claimed');
|
|
723
|
+
const all = await t.store.list({ limit: 100, offset: 0 });
|
|
724
|
+
equal(all.filter((i) => i.runs !== 1 || i.leaseOwner !== owners.get(i.id)).map((i) => i.id), [], 'each claimed once, by its owner');
|
|
725
|
+
});
|
|
726
|
+
add('concurrent claims never lease more than a limit allows, and fill every free slot', async (t) => {
|
|
727
|
+
for (let i = 0; i < 60; i++) {
|
|
728
|
+
await t.keyed(`k${i % 6}-${String(i).padStart(2, '0')}`, `k${i % 6}`, i);
|
|
729
|
+
}
|
|
730
|
+
const limits = [{ workflow: W.name, limit: 4, perKey: 1 }];
|
|
731
|
+
const tokens = new Map();
|
|
732
|
+
for (let round = 0; round < 5; round++) {
|
|
733
|
+
const now = 1_000 + round;
|
|
734
|
+
await Promise.all(['c1', 'c2', 'c3', 'c4', 'c5', 'c6'].map(async (owner) => {
|
|
735
|
+
await jitter();
|
|
736
|
+
const token = randomUUID();
|
|
737
|
+
for (const instance of (await t.claim(now, { owner, token, limit: 3, leaseUntil: FAR, limits })).instances) {
|
|
738
|
+
tokens.set(instance.id, token);
|
|
739
|
+
}
|
|
740
|
+
}));
|
|
741
|
+
const leased = (await t.store.list({ limit: 100, offset: 0 })).filter((i) => i.leaseUntil !== null && i.leaseUntil >= now);
|
|
742
|
+
equal(leased.length, 4, `round ${round}: every slot filled, none more`);
|
|
743
|
+
equal(new Set(leased.map((i) => i.concurrencyKey)).size, 4, `round ${round}: one per key`);
|
|
744
|
+
await Promise.all(leased.map((i) => jitter().then(() => t.store.write(i.id, tokens.get(i.id), { now, entries: [], status: 'completed', error: null, release: { wakeAt: null, waits: [], signalCursor: 0 } }))));
|
|
745
|
+
}
|
|
746
|
+
});
|
|
747
|
+
add('concurrent claims never start more than a rate limit allows in a window, and fill its room', async (t) => {
|
|
748
|
+
for (let i = 0; i < 90; i++) {
|
|
749
|
+
await t.prioritized(`r${String(i).padStart(2, '0')}`, i, { rateLimitKey: `k${i % 3}` });
|
|
750
|
+
}
|
|
751
|
+
// Per key: 2 per window of 1000; overall: 5. Each round is a new window. Concurrent claims of one workflow
|
|
752
|
+
// may leave room (a claim passes over the instances another is taking): claims after them take the rest.
|
|
753
|
+
const rateLimits = [{ workflow: W.name, limit: { max: 5, duration: 1_000 }, perKey: { max: 2, duration: 1_000 } }];
|
|
754
|
+
const started = new Set();
|
|
755
|
+
for (let round = 0; round < 5; round++) {
|
|
756
|
+
const now = 10_000 * (round + 1);
|
|
757
|
+
const claim = async (owner) => {
|
|
758
|
+
const token = randomUUID();
|
|
759
|
+
return (await t.claim(now, { owner, token, limit: 2, leaseUntil: now + 1, rateLimits })).instances.map((instance) => ({ instance, token }));
|
|
760
|
+
};
|
|
761
|
+
const claimed = (await Promise.all(['c1', 'c2', 'c3', 'c4', 'c5', 'c6'].map((owner) => jitter().then(() => claim(owner))))).flat();
|
|
762
|
+
for (let more = await claim('c7'); more.length > 0; more = await claim('c7')) {
|
|
763
|
+
claimed.push(...more);
|
|
764
|
+
}
|
|
765
|
+
equal(claimed.length, 5, `round ${round}: the window's room filled, no more`);
|
|
766
|
+
const perKey = new Map();
|
|
767
|
+
for (const { instance } of claimed) {
|
|
768
|
+
perKey.set(instance.rateLimitKey, (perKey.get(instance.rateLimitKey) ?? 0) + 1);
|
|
769
|
+
if (started.has(instance.id)) {
|
|
770
|
+
throw new Error(`${instance.id} was claimed twice`);
|
|
771
|
+
}
|
|
772
|
+
started.add(instance.id);
|
|
773
|
+
}
|
|
774
|
+
equal([...perKey.values()].every((n) => n <= 2), true, `round ${round}: at most 2 per key`);
|
|
775
|
+
await Promise.all(claimed.map(({ instance, token }) => t.store.write(instance.id, token, { now, entries: [], status: 'completed', error: null, release: { wakeAt: null, waits: [], signalCursor: 0 } })));
|
|
776
|
+
}
|
|
777
|
+
});
|
|
778
|
+
scheduleCases((name, run) => add(name, (t) => run(workflowScheduleStore(() => t.store))), 'concurrent');
|
|
779
|
+
add('purge() racing claims that reuse ended rate-limit windows neither deadlocks nor lets a window overflow', async (t) => {
|
|
780
|
+
for (let i = 0; i < 40; i++) {
|
|
781
|
+
await t.prioritized(`w${String(i).padStart(2, '0')}`, i, { rateLimitKey: `k${i % 8}` });
|
|
782
|
+
}
|
|
783
|
+
// Every round's windows (500ms) have ended by the next round, when purges delete them (in the order they ended,
|
|
784
|
+
// not by key, as the claimers open them a millisecond apart) while claims reuse them (by key).
|
|
785
|
+
const rateLimits = [{ workflow: W.name, limit: null, perKey: { max: 1, duration: 500 } }];
|
|
786
|
+
for (let round = 0; round < 5; round++) {
|
|
787
|
+
const now = 1_000 * (round + 1);
|
|
788
|
+
const claims = ['c1', 'c2', 'c3', 'c4'].map(async (owner, i) => {
|
|
789
|
+
const token = randomUUID();
|
|
790
|
+
await jitter();
|
|
791
|
+
return (await t.claim(now + i, { owner, token, limit: 3, leaseUntil: now + 10, rateLimits })).instances.map((instance) => ({ instance, token }));
|
|
792
|
+
});
|
|
793
|
+
const purges = [1, 2].map(async () => {
|
|
794
|
+
await jitter();
|
|
795
|
+
return t.store.purge({ statuses: ['completed'], before: now, limit: 2 });
|
|
796
|
+
});
|
|
797
|
+
const claimed = (await Promise.all([...claims, ...purges.map((purge) => purge.then(() => []))])).flat();
|
|
798
|
+
const keys = claimed.map(({ instance }) => instance.rateLimitKey);
|
|
799
|
+
equal(new Set(keys).size, keys.length, `round ${round}: at most one per key's window`);
|
|
800
|
+
await Promise.all(claimed.map(({ instance, token }) => t.store.write(instance.id, token, { now, entries: [], status: 'completed', error: null, release: { wakeAt: null, waits: [], signalCursor: 0 } })));
|
|
801
|
+
}
|
|
802
|
+
});
|
|
803
|
+
add('a signal racing a suspension never loses the wake-up', async (t) => {
|
|
804
|
+
const n = 60;
|
|
805
|
+
for (let i = 0; i < n; i++) {
|
|
806
|
+
await t.create(`r${i}`);
|
|
807
|
+
}
|
|
808
|
+
const { instances } = await t.claim(1, { limit: n, leaseUntil: FAR, token: 't' });
|
|
809
|
+
equal(instances.length, n, 'claimed');
|
|
810
|
+
await Promise.all(instances.map(async (instance, i) => {
|
|
811
|
+
const { lastSignalId: cursor } = await t.claim(0); // what the execution saw when it started
|
|
812
|
+
const suspend = async () => {
|
|
813
|
+
await jitter();
|
|
814
|
+
const ok = await t.store.write(instance.id, 't', {
|
|
815
|
+
now: 5,
|
|
816
|
+
entries: [entry('wait', { kind: 'signal', status: 'pending', attempts: 0, wakeAt: FAR })],
|
|
817
|
+
status: 'suspended',
|
|
818
|
+
release: { wakeAt: FAR, waits: [{ signal: 'go', key: instance.id }], signalCursor: cursor },
|
|
819
|
+
});
|
|
820
|
+
if (!ok) {
|
|
821
|
+
throw new Error(`suspending ${instance.id} was refused`);
|
|
822
|
+
}
|
|
823
|
+
};
|
|
824
|
+
const signal = async () => {
|
|
825
|
+
await jitter();
|
|
826
|
+
const data = { name: 'go', key: instance.id, dedupeId: null, payload: i, now: 6 };
|
|
827
|
+
if (i % 2 === 0 || !transaction || !t.store.signalInTransaction) {
|
|
828
|
+
return t.store.signal(data);
|
|
829
|
+
}
|
|
830
|
+
// In the application's transaction, which holds the lock a little longer.
|
|
831
|
+
return transaction(async (tx) => {
|
|
832
|
+
const result = await t.store.signalInTransaction(tx, data);
|
|
833
|
+
await jitter();
|
|
834
|
+
return result;
|
|
835
|
+
});
|
|
836
|
+
};
|
|
837
|
+
await Promise.all([suspend(), signal()]);
|
|
838
|
+
}));
|
|
839
|
+
const parked = (await t.store.list({ limit: n, offset: 0 })).filter((i) => i.wakeAt === FAR).map((i) => i.id);
|
|
840
|
+
equal(parked, [], 'every instance was woken by its signal (wakeAt 6) or saw it while suspending (5)');
|
|
841
|
+
});
|
|
842
|
+
add("a child's ending write racing its parent's suspension never loses the wake-up", async (t) => {
|
|
843
|
+
const n = 30;
|
|
844
|
+
for (let i = 0; i < n; i++) {
|
|
845
|
+
await t.create(`parent-${i}`);
|
|
846
|
+
await t.create(`child-${i}`);
|
|
847
|
+
}
|
|
848
|
+
const { instances } = await t.claim(1, { limit: 2 * n, leaseUntil: FAR, token: 't' });
|
|
849
|
+
equal(instances.length, 2 * n, 'claimed');
|
|
850
|
+
await Promise.all(Array.from({ length: n }, async (_, i) => {
|
|
851
|
+
const { lastSignalId: cursor } = await t.claim(0);
|
|
852
|
+
const suspend = async () => {
|
|
853
|
+
await jitter();
|
|
854
|
+
const release = { wakeAt: FAR, waits: [{ signal: 'child.ended', key: `child-${i}` }], signalCursor: cursor };
|
|
855
|
+
if (!(await t.store.write(`parent-${i}`, 't', { now: 5, entries: [], status: 'suspended', release }))) {
|
|
856
|
+
throw new Error(`suspending parent-${i} was refused`);
|
|
857
|
+
}
|
|
858
|
+
};
|
|
859
|
+
const end = async () => {
|
|
860
|
+
await jitter();
|
|
861
|
+
const signal = { name: 'child.ended', key: `child-${i}`, dedupeId: `child-${i}`, payload: i, now: 6 };
|
|
862
|
+
const release = { wakeAt: null, waits: [], signalCursor: cursor };
|
|
863
|
+
if (!(await t.store.write(`child-${i}`, 't', { now: 6, entries: [], status: 'completed', output: i, error: null, release, signal }))) {
|
|
864
|
+
throw new Error(`ending child-${i} was refused`);
|
|
865
|
+
}
|
|
866
|
+
};
|
|
867
|
+
await Promise.all([suspend(), end()]);
|
|
868
|
+
}));
|
|
869
|
+
const parked = (await t.store.list({ limit: 2 * n, offset: 0 })).filter((i) => i.wakeAt === FAR).map((i) => i.id);
|
|
870
|
+
equal(parked, [], "every parent was woken by its child's end (wakeAt 6) or saw it while suspending (5)");
|
|
871
|
+
});
|
|
872
|
+
add('signal ids become visible in id order: nothing below a cursor shows up later', async (t) => {
|
|
873
|
+
const probes = [];
|
|
874
|
+
let sending = true;
|
|
875
|
+
const probe = async () => {
|
|
876
|
+
while (sending) {
|
|
877
|
+
const { lastSignalId: cursor } = await t.claim(0);
|
|
878
|
+
const seen = (await t.store.signals({ name: 's', key: null, afterId: 0, upToId: cursor })).map((s) => s.id);
|
|
879
|
+
probes.push({ cursor, seen });
|
|
880
|
+
await jitter();
|
|
881
|
+
}
|
|
882
|
+
};
|
|
883
|
+
const send = async (i) => {
|
|
884
|
+
await jitter();
|
|
885
|
+
if (i % 3 !== 0 || !transaction || !t.store.signalInTransaction) {
|
|
886
|
+
return t.store.signal({ name: 's', key: null, dedupeId: null, payload: i, now: 1 });
|
|
887
|
+
}
|
|
888
|
+
return transaction(async (tx) => {
|
|
889
|
+
const result = await t.store.signalInTransaction(tx, { name: 's', key: null, dedupeId: null, payload: i, now: 1 });
|
|
890
|
+
await jitter();
|
|
891
|
+
return result;
|
|
892
|
+
});
|
|
893
|
+
};
|
|
894
|
+
const probing = [probe(), probe()];
|
|
895
|
+
const sent = await Promise.all(Array.from({ length: 60 }, (_, i) => send(i)));
|
|
896
|
+
sending = false;
|
|
897
|
+
await Promise.all(probing);
|
|
898
|
+
equal(new Set(sent.map((s) => s.id)).size, 60, 'unique ids');
|
|
899
|
+
const all = (await t.store.signals({ name: 's', key: null, afterId: 0, upToId: Number.MAX_SAFE_INTEGER })).map((s) => s.id);
|
|
900
|
+
for (const { cursor, seen } of probes) {
|
|
901
|
+
equal(all.filter((id) => id <= cursor), seen, `the signals at or below cursor ${cursor}`);
|
|
902
|
+
}
|
|
903
|
+
});
|
|
904
|
+
add('concurrent signals with one dedupeId store it once, and every call returns its id', async (t) => {
|
|
905
|
+
const send = async (i) => {
|
|
906
|
+
await jitter();
|
|
907
|
+
const data = { name: 'payment.captured', key: 'o1', dedupeId: 'ch_1', payload: i, now: i };
|
|
908
|
+
if (i % 2 === 0 || !transaction || !t.store.signalInTransaction) {
|
|
909
|
+
return t.store.signal(data);
|
|
910
|
+
}
|
|
911
|
+
return transaction(async (tx) => {
|
|
912
|
+
const result = await t.store.signalInTransaction(tx, data);
|
|
913
|
+
await jitter();
|
|
914
|
+
return result;
|
|
915
|
+
});
|
|
916
|
+
};
|
|
917
|
+
const results = await Promise.all(Array.from({ length: 12 }, (_, i) => send(i)));
|
|
918
|
+
equal(results.filter((r) => r.created).length, 1, 'created once');
|
|
919
|
+
equal(new Set(results.map((r) => r.id)).size, 1, 'one id for every call');
|
|
920
|
+
const stored = await t.store.signals({ name: 'payment.captured', key: 'o1', afterId: 0, upToId: Number.MAX_SAFE_INTEGER });
|
|
921
|
+
equal(stored.map((s) => s.id), [results[0].id], 'one signal stored');
|
|
922
|
+
});
|
|
923
|
+
add('a cancel racing a suspension leaves the instance due', async (t) => {
|
|
924
|
+
const n = 40;
|
|
925
|
+
for (let i = 0; i < n; i++) {
|
|
926
|
+
await t.create(`c${i}`);
|
|
927
|
+
}
|
|
928
|
+
const { instances } = await t.claim(1, { limit: n, leaseUntil: FAR, token: 't' });
|
|
929
|
+
await Promise.all(instances.map(async (instance) => {
|
|
930
|
+
await Promise.all([
|
|
931
|
+
jitter().then(() => t.store.requestCancel(instance.id, { reason: 'stop', now: 3, terminate: false })),
|
|
932
|
+
jitter().then(() => t.store.write(instance.id, 't', { now: 4, entries: [], status: 'suspended', release: { wakeAt: FAR, waits: [], signalCursor: 0 } })),
|
|
933
|
+
]);
|
|
934
|
+
}));
|
|
935
|
+
const all = await t.store.list({ limit: n, offset: 0 });
|
|
936
|
+
equal(all.filter((i) => !i.cancelRequested || i.wakeAt === FAR).map((i) => i.id), [], 'cancelled and due (wakeAt 3 or 4)');
|
|
937
|
+
});
|
|
938
|
+
add('concurrent create(), requestCancel() and terminates of one id: one wins', async (t) => {
|
|
939
|
+
const created = await Promise.all(Array.from({ length: 8 }, (_, i) => jitter().then(() => t.store.create({ id: 'same', workflow: 'w', version: 1, input: i, deadline: null, now: i }))));
|
|
940
|
+
equal(created.filter((r) => r.created).length, 1, 'created once');
|
|
941
|
+
const winner = created.find((r) => r.created).instance;
|
|
942
|
+
equal(created.filter((r) => !isDeepStrictEqual(r.instance.input, winner.input)).length, 0, 'every call returns the stored instance');
|
|
943
|
+
const accepted = await Promise.all(Array.from({ length: 8 }, (_, i) => jitter().then(() => t.store.requestCancel('same', { reason: `r${i}`, now: 10, terminate: false }))));
|
|
944
|
+
equal(accepted.filter(Boolean).length, 1, 'one cancel accepted');
|
|
945
|
+
const terminated = await Promise.all(Array.from({ length: 8 }, (_, i) => jitter().then(() => t.store.requestCancel('same', { reason: `t${i}`, now: 11, terminate: true }))));
|
|
946
|
+
equal(terminated.filter(Boolean).length, 1, 'one terminate accepted');
|
|
947
|
+
});
|
|
948
|
+
add('purge() racing finishing, starting and signalling instances deletes only what was finished and unreachable', async (t) => {
|
|
949
|
+
const n = 30;
|
|
950
|
+
for (let i = 0; i < n; i++) {
|
|
951
|
+
await t.create(`p${i}`);
|
|
952
|
+
}
|
|
953
|
+
await t.claim(1, { limit: n, token: 't', leaseUntil: FAR });
|
|
954
|
+
for (let i = 0; i < 10; i++) {
|
|
955
|
+
await t.store.signal({ name: 'old', key: null, dedupeId: null, payload: i, now: 0 });
|
|
956
|
+
}
|
|
957
|
+
let purging = true;
|
|
958
|
+
const purger = async () => {
|
|
959
|
+
while (purging) {
|
|
960
|
+
await t.store.purge({ statuses: ['completed'], before: 1_000, limit: 3 });
|
|
961
|
+
await jitter();
|
|
962
|
+
}
|
|
963
|
+
};
|
|
964
|
+
const work = Array.from({ length: n }, async (_, i) => {
|
|
965
|
+
await jitter();
|
|
966
|
+
if (i % 2 === 0) {
|
|
967
|
+
await t.store.write(`p${i}`, 't', { now: 10, entries: [], status: 'completed', error: null, release: { wakeAt: null, waits: [], signalCursor: 0 } });
|
|
968
|
+
}
|
|
969
|
+
else {
|
|
970
|
+
await t.store.create({ id: `new${i}`, workflow: W.name, version: W.version, input: null, deadline: null, now: 2_000 });
|
|
971
|
+
await t.store.signal({ name: 'new', key: null, dedupeId: null, payload: i, now: 2_000 });
|
|
972
|
+
}
|
|
973
|
+
});
|
|
974
|
+
const purgers = [purger(), purger()];
|
|
975
|
+
await Promise.all(work);
|
|
976
|
+
purging = false;
|
|
977
|
+
await Promise.all(purgers);
|
|
978
|
+
while ((await t.store.purge({ statuses: ['completed'], before: 1_000, limit: 3 })).instances > 0) {
|
|
979
|
+
// the rest of the finished ones
|
|
980
|
+
}
|
|
981
|
+
const left = (await t.store.list({ limit: 100, offset: 0 })).map((i) => i.id);
|
|
982
|
+
equal(left.filter((id) => id.startsWith('p')).length, n / 2, 'every unfinished instance kept, every finished one purged');
|
|
983
|
+
equal(left.filter((id) => id.startsWith('new')).length, n / 2, 'every new instance kept');
|
|
984
|
+
equal((await t.store.signals({ name: 'new', key: null, afterId: 0, upToId: FAR })).length, n / 2, 'every recent signal kept');
|
|
985
|
+
});
|
|
986
|
+
add('concurrent reopens of one read accept one, and a purge racing them never deletes a reopened instance', async (t) => {
|
|
987
|
+
const n = 20;
|
|
988
|
+
for (let i = 0; i < n; i++) {
|
|
989
|
+
await t.create(`r${i}`);
|
|
990
|
+
}
|
|
991
|
+
await t.claim(1, { limit: n, token: 't', leaseUntil: FAR });
|
|
992
|
+
for (let i = 0; i < n; i++) {
|
|
993
|
+
await t.store.write(`r${i}`, 't', { now: 2, entries: [], status: 'failed', error: null, release: { wakeAt: null, waits: [], signalCursor: 0 } });
|
|
994
|
+
}
|
|
995
|
+
const reopen = (id) => jitter().then(() => t.store.reopen(id, { expect: { status: 'failed', runs: 1 }, status: 'pending', error: null, entries: [], now: 3 }));
|
|
996
|
+
const results = await Promise.all(Array.from({ length: n }, async (_, i) => {
|
|
997
|
+
const [accepted] = await Promise.all([
|
|
998
|
+
Promise.all([reopen(`r${i}`), reopen(`r${i}`)]),
|
|
999
|
+
jitter().then(() => t.store.purge({ statuses: ['failed'], before: 100, limit: 1 })),
|
|
1000
|
+
]);
|
|
1001
|
+
return accepted;
|
|
1002
|
+
}));
|
|
1003
|
+
for (const [i, [first, second]] of results.entries()) {
|
|
1004
|
+
const instance = await t.store.get(`r${i}`);
|
|
1005
|
+
if (first && second) {
|
|
1006
|
+
throw new Error(`r${i} was reopened twice`);
|
|
1007
|
+
}
|
|
1008
|
+
if ((first || second) !== (instance?.status === 'pending')) {
|
|
1009
|
+
throw new Error(`r${i}: reopened ${first || second}, but ${instance ? `is ${instance.status}` : 'was purged'}`);
|
|
1010
|
+
}
|
|
1011
|
+
}
|
|
1012
|
+
});
|
|
1013
|
+
add("a stale lease holder's writes never land, however they interleave with the new holder's", async (t) => {
|
|
1014
|
+
const n = 20;
|
|
1015
|
+
for (let i = 0; i < n; i++) {
|
|
1016
|
+
await t.create(`f${i}`);
|
|
1017
|
+
}
|
|
1018
|
+
await t.claim(1, { limit: n, token: 'old', leaseUntil: 10 });
|
|
1019
|
+
await t.claim(11, { limit: n, token: 'new', owner: 'w2', leaseUntil: FAR });
|
|
1020
|
+
const results = await Promise.all(Array.from({ length: n }, (_, i) => `f${i}`).flatMap((id) => [
|
|
1021
|
+
jitter().then(() => t.store.write(id, 'old', { now: 12, entries: [entry('stale')], status: 'compensating', error: { name: 'E', message: 'stale' } })),
|
|
1022
|
+
jitter().then(() => t.store.write(id, 'new', t.journalWrite([entry('fresh')]))),
|
|
1023
|
+
jitter().then(() => t.store.write(id, 'old', { now: 12, entries: [], status: 'failed', release: { wakeAt: null, waits: [], signalCursor: 0 } })),
|
|
1024
|
+
]));
|
|
1025
|
+
equal(results.filter((ok, i) => ok !== (i % 3 === 1)).length, 0, 'only the new holder writes');
|
|
1026
|
+
for (let i = 0; i < n; i++) {
|
|
1027
|
+
expect(await t.store.get(`f${i}`, { journal: true }), { status: 'running', leaseUntil: FAR, error: null, journal: [{ name: 'fresh' }] }, `f${i}`);
|
|
1028
|
+
}
|
|
1029
|
+
});
|
|
1030
|
+
}
|
|
1031
|
+
return cases;
|
|
1032
|
+
}
|
|
1033
|
+
// ---------------------------------------------------------------- helpers
|
|
1034
|
+
const FAR = 9_000_000_000_000; // a deadline nobody reaches in a test
|
|
1035
|
+
const W = { name: 'order-fulfilment', version: 1 };
|
|
1036
|
+
class Harness {
|
|
1037
|
+
constructor(store) {
|
|
1038
|
+
this.store = store;
|
|
1039
|
+
}
|
|
1040
|
+
create(id, now = 0, workflow = 'order-fulfilment', version = 1) {
|
|
1041
|
+
return this.store.create({ id, workflow, version, input: { id }, deadline: null, now });
|
|
1042
|
+
}
|
|
1043
|
+
claim(now, o = {}) {
|
|
1044
|
+
return this.store.claim({
|
|
1045
|
+
owner: o.owner ?? 'w1',
|
|
1046
|
+
token: o.token ?? randomUUID(),
|
|
1047
|
+
now,
|
|
1048
|
+
leaseUntil: o.leaseUntil ?? now + 1_000,
|
|
1049
|
+
limit: o.limit ?? 100,
|
|
1050
|
+
workflows: o.workflows ?? [W],
|
|
1051
|
+
...(o.limits ? { limits: o.limits } : {}),
|
|
1052
|
+
...(o.rateLimits ? { rateLimits: o.rateLimits } : {}),
|
|
1053
|
+
});
|
|
1054
|
+
}
|
|
1055
|
+
/** Claims at `now` and finishes what it claimed (so nothing comes back when its lease ends); returns the ids. */
|
|
1056
|
+
async claimAndFinish(now, o = {}) {
|
|
1057
|
+
const token = randomUUID();
|
|
1058
|
+
const { instances } = await this.claim(now, { ...o, token });
|
|
1059
|
+
for (const instance of instances) {
|
|
1060
|
+
await this.store.write(instance.id, token, { now, entries: [], status: 'completed', error: null, release: { wakeAt: null, waits: [], signalCursor: 0 } });
|
|
1061
|
+
}
|
|
1062
|
+
return instances.map((instance) => instance.id);
|
|
1063
|
+
}
|
|
1064
|
+
/** An instance of `W` with a priority and a rate-limit key. */
|
|
1065
|
+
prioritized(id, now, o = {}) {
|
|
1066
|
+
return this.store.create({ id, workflow: W.name, version: 1, input: { id }, deadline: null, now, ...o });
|
|
1067
|
+
}
|
|
1068
|
+
/** An instance of `W` with a concurrency key. */
|
|
1069
|
+
keyed(id, key, now, workflow = W.name) {
|
|
1070
|
+
return this.store.create({ id, workflow, version: 1, input: { id }, deadline: null, concurrencyKey: key, now });
|
|
1071
|
+
}
|
|
1072
|
+
/** Ends a lease: the instance is parked, and holds no slot. */
|
|
1073
|
+
release(id, token) {
|
|
1074
|
+
return this.store.write(id, token, { now: 1, entries: [], status: 'suspended', release: { wakeAt: FAR, waits: [], signalCursor: 0 } });
|
|
1075
|
+
}
|
|
1076
|
+
journalWrite(entries) {
|
|
1077
|
+
return { now: 1, entries };
|
|
1078
|
+
}
|
|
1079
|
+
}
|
|
1080
|
+
function entry(name, extra = {}) {
|
|
1081
|
+
return { name, kind: 'step', status: 'completed', attempts: 1, updatedAt: 1, ...extra };
|
|
1082
|
+
}
|
|
1083
|
+
function requireTransactionMethods(store) {
|
|
1084
|
+
const { createInTransaction, signalInTransaction } = store;
|
|
1085
|
+
if (typeof createInTransaction !== 'function' || typeof signalInTransaction !== 'function') {
|
|
1086
|
+
throw new Error('options.transaction is set, but the store lacks createInTransaction() or signalInTransaction().');
|
|
1087
|
+
}
|
|
1088
|
+
return { createInTransaction: createInTransaction.bind(store), signalInTransaction: signalInTransaction.bind(store) };
|
|
1089
|
+
}
|