@dudousxd/nestjs-durable-testing 0.20.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +63 -1
- package/dist/index.js +809 -2
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -375,7 +375,50 @@ function createTestEngine() {
|
|
|
375
375
|
__name(createTestEngine, "createTestEngine");
|
|
376
376
|
|
|
377
377
|
// src/replay.ts
|
|
378
|
-
import { InMemoryStateStore as InMemoryStateStore2, WorkflowEngine as WorkflowEngine2 } from "@dudousxd/nestjs-durable-core";
|
|
378
|
+
import { InMemoryStateStore as InMemoryStateStore2, RUN_HISTORY_FIXTURE_VERSION, WorkflowEngine as WorkflowEngine2 } from "@dudousxd/nestjs-durable-core";
|
|
379
|
+
async function captureHistory(source, runId) {
|
|
380
|
+
const run2 = await source.getRun(runId);
|
|
381
|
+
if (!run2) return null;
|
|
382
|
+
return {
|
|
383
|
+
run: run2,
|
|
384
|
+
checkpoints: await source.listCheckpoints(runId)
|
|
385
|
+
};
|
|
386
|
+
}
|
|
387
|
+
__name(captureHistory, "captureHistory");
|
|
388
|
+
function serializeRunHistory(history) {
|
|
389
|
+
return `${JSON.stringify({
|
|
390
|
+
version: RUN_HISTORY_FIXTURE_VERSION,
|
|
391
|
+
...history
|
|
392
|
+
}, null, 2)}
|
|
393
|
+
`;
|
|
394
|
+
}
|
|
395
|
+
__name(serializeRunHistory, "serializeRunHistory");
|
|
396
|
+
function parseRunHistory(json) {
|
|
397
|
+
const raw = JSON.parse(json);
|
|
398
|
+
if (!raw || typeof raw !== "object" || !raw.run || !Array.isArray(raw.checkpoints)) {
|
|
399
|
+
throw new Error("not a run history fixture: expected { run, checkpoints }");
|
|
400
|
+
}
|
|
401
|
+
if (raw.version !== void 0 && raw.version !== RUN_HISTORY_FIXTURE_VERSION) {
|
|
402
|
+
throw new Error(`unsupported run history fixture version ${JSON.stringify(raw.version)} (this kit reads version ${RUN_HISTORY_FIXTURE_VERSION}); re-export the fixture or upgrade @dudousxd/nestjs-durable-testing`);
|
|
403
|
+
}
|
|
404
|
+
const date = /* @__PURE__ */ __name((v) => new Date(v), "date");
|
|
405
|
+
const run2 = {
|
|
406
|
+
...raw.run,
|
|
407
|
+
createdAt: date(raw.run.createdAt),
|
|
408
|
+
updatedAt: date(raw.run.updatedAt)
|
|
409
|
+
};
|
|
410
|
+
const checkpoints = raw.checkpoints.map((cp2) => ({
|
|
411
|
+
...cp2,
|
|
412
|
+
enqueuedAt: date(cp2.enqueuedAt),
|
|
413
|
+
startedAt: date(cp2.startedAt),
|
|
414
|
+
finishedAt: date(cp2.finishedAt)
|
|
415
|
+
}));
|
|
416
|
+
return {
|
|
417
|
+
run: run2,
|
|
418
|
+
checkpoints
|
|
419
|
+
};
|
|
420
|
+
}
|
|
421
|
+
__name(parseRunHistory, "parseRunHistory");
|
|
379
422
|
async function assertReplayable(register, history) {
|
|
380
423
|
const store = new InMemoryStateStore2();
|
|
381
424
|
await store.createRun({
|
|
@@ -2314,6 +2357,18 @@ function runStateStoreContract(name, makeStore) {
|
|
|
2314
2357
|
await store.releaseRunLock("r1");
|
|
2315
2358
|
expect3(await store.tryLockRun("r1", "C", 9e3, 2600)).toBe(true);
|
|
2316
2359
|
});
|
|
2360
|
+
t("releaseRunLock with an owner clears only a lease that owner still holds", async () => {
|
|
2361
|
+
await store.createRun(run({
|
|
2362
|
+
id: "r1"
|
|
2363
|
+
}));
|
|
2364
|
+
expect3(await store.tryLockRun("r1", "A", 2e3, 1e3)).toBe(true);
|
|
2365
|
+
expect3(await store.tryLockRun("r1", "B", 4e3, 2500)).toBe(true);
|
|
2366
|
+
await store.releaseRunLock("r1", "A");
|
|
2367
|
+
expect3(await store.tryLockRun("r1", "C", 9e3, 3e3)).toBe(false);
|
|
2368
|
+
expect3((await store.getRun("r1"))?.lockedBy).toBe("B");
|
|
2369
|
+
await store.releaseRunLock("r1", "B");
|
|
2370
|
+
expect3(await store.tryLockRun("r1", "C", 9e3, 3e3)).toBe(true);
|
|
2371
|
+
});
|
|
2317
2372
|
t("renewRunLock only succeeds for the current owner", async () => {
|
|
2318
2373
|
await store.createRun(run({
|
|
2319
2374
|
id: "r1"
|
|
@@ -3761,6 +3816,42 @@ function runStateStoreContract(name, makeStore) {
|
|
|
3761
3816
|
});
|
|
3762
3817
|
expect3(await store.listCheckpoints("other")).toHaveLength(1);
|
|
3763
3818
|
});
|
|
3819
|
+
t("getRuns returns the existing runs among the given ids", async () => {
|
|
3820
|
+
expect3(store.getRuns).toBeTypeOf("function");
|
|
3821
|
+
await store.createRun(run({
|
|
3822
|
+
id: "p.all.0.0",
|
|
3823
|
+
status: "completed",
|
|
3824
|
+
output: {
|
|
3825
|
+
n: 1
|
|
3826
|
+
}
|
|
3827
|
+
}));
|
|
3828
|
+
await store.createRun(run({
|
|
3829
|
+
id: "p.all.0.1",
|
|
3830
|
+
status: "suspended"
|
|
3831
|
+
}));
|
|
3832
|
+
await store.createRun(run({
|
|
3833
|
+
id: "other"
|
|
3834
|
+
}));
|
|
3835
|
+
const got = await store.getRuns?.([
|
|
3836
|
+
"p.all.0.0",
|
|
3837
|
+
"p.all.0.1",
|
|
3838
|
+
"p.all.0.2",
|
|
3839
|
+
"p.all.0.0"
|
|
3840
|
+
]) ?? [];
|
|
3841
|
+
expect3(got.map((r) => r.id).sort()).toEqual([
|
|
3842
|
+
"p.all.0.0",
|
|
3843
|
+
"p.all.0.1"
|
|
3844
|
+
]);
|
|
3845
|
+
const done = got.find((r) => r.id === "p.all.0.0");
|
|
3846
|
+
expect3(done?.status).toBe("completed");
|
|
3847
|
+
expect3(done?.output).toEqual({
|
|
3848
|
+
n: 1
|
|
3849
|
+
});
|
|
3850
|
+
expect3(await store.getRuns?.([])).toEqual([]);
|
|
3851
|
+
expect3(await store.getRuns?.([
|
|
3852
|
+
"missing"
|
|
3853
|
+
])).toEqual([]);
|
|
3854
|
+
});
|
|
3764
3855
|
t("listSignalWaitersByRun returns only the given runs' waiters", async () => {
|
|
3765
3856
|
expect3(store.listSignalWaitersByRun).toBeTypeOf("function");
|
|
3766
3857
|
await store.putSignalWaiter({
|
|
@@ -4236,9 +4327,721 @@ async function assertTransportConformance(transport, idPrefix = "conf") {
|
|
|
4236
4327
|
}
|
|
4237
4328
|
}
|
|
4238
4329
|
__name(assertTransportConformance, "assertTransportConformance");
|
|
4330
|
+
|
|
4331
|
+
// src/run-dispatch-contract.ts
|
|
4332
|
+
import { InMemoryStateStore as InMemoryStateStore4, WorkflowEngine as WorkflowEngine5 } from "@dudousxd/nestjs-durable-core";
|
|
4333
|
+
import { describe as describe3, expect as expect4, it as it3 } from "vitest";
|
|
4334
|
+
function recorded(inner) {
|
|
4335
|
+
const batches = [];
|
|
4336
|
+
const failing = {
|
|
4337
|
+
on: false
|
|
4338
|
+
};
|
|
4339
|
+
const transport = new Proxy(inner, {
|
|
4340
|
+
get(target, prop, receiver) {
|
|
4341
|
+
if (prop === "dispatchRuns") {
|
|
4342
|
+
return async (runs, opts) => {
|
|
4343
|
+
if (failing.on) throw new Error("broker down");
|
|
4344
|
+
batches.push(runs);
|
|
4345
|
+
return target.dispatchRuns?.(runs, opts);
|
|
4346
|
+
};
|
|
4347
|
+
}
|
|
4348
|
+
const value = Reflect.get(target, prop, receiver);
|
|
4349
|
+
return typeof value === "function" ? value.bind(target) : value;
|
|
4350
|
+
}
|
|
4351
|
+
});
|
|
4352
|
+
return {
|
|
4353
|
+
transport,
|
|
4354
|
+
batches,
|
|
4355
|
+
failing
|
|
4356
|
+
};
|
|
4357
|
+
}
|
|
4358
|
+
__name(recorded, "recorded");
|
|
4359
|
+
async function until(check, timeoutMs) {
|
|
4360
|
+
const deadline = Date.now() + timeoutMs;
|
|
4361
|
+
while (Date.now() < deadline) {
|
|
4362
|
+
if (await check()) return;
|
|
4363
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
4364
|
+
}
|
|
4365
|
+
throw new Error("condition not met");
|
|
4366
|
+
}
|
|
4367
|
+
__name(until, "until");
|
|
4368
|
+
var pause = /* @__PURE__ */ __name((ms2) => new Promise((resolve) => setTimeout(resolve, ms2)), "pause");
|
|
4369
|
+
var FANS = [
|
|
4370
|
+
{
|
|
4371
|
+
fan: "ctx.all fan",
|
|
4372
|
+
settle: false
|
|
4373
|
+
},
|
|
4374
|
+
{
|
|
4375
|
+
fan: "ctx.allSettled fan",
|
|
4376
|
+
settle: true
|
|
4377
|
+
}
|
|
4378
|
+
];
|
|
4379
|
+
function instances(store, transports, clock) {
|
|
4380
|
+
return transports.map((transport, i) => {
|
|
4381
|
+
const engine = new WorkflowEngine5({
|
|
4382
|
+
store,
|
|
4383
|
+
transport,
|
|
4384
|
+
reconcileMs: 0,
|
|
4385
|
+
...clock ? {
|
|
4386
|
+
clock
|
|
4387
|
+
} : {}
|
|
4388
|
+
});
|
|
4389
|
+
return {
|
|
4390
|
+
name: `instance-${i}`,
|
|
4391
|
+
engine
|
|
4392
|
+
};
|
|
4393
|
+
});
|
|
4394
|
+
}
|
|
4395
|
+
__name(instances, "instances");
|
|
4396
|
+
function registerFan(all, fanSize, ran, settle2) {
|
|
4397
|
+
for (const { name, engine } of all) {
|
|
4398
|
+
engine.register("child", "1", async (ctx, input) => {
|
|
4399
|
+
const { i } = input;
|
|
4400
|
+
return ctx.localStep("work", async () => {
|
|
4401
|
+
ran.set(i, [
|
|
4402
|
+
...ran.get(i) ?? [],
|
|
4403
|
+
name
|
|
4404
|
+
]);
|
|
4405
|
+
await pause(15);
|
|
4406
|
+
return i * 2;
|
|
4407
|
+
});
|
|
4408
|
+
});
|
|
4409
|
+
engine.register("parent", "1", async (ctx) => {
|
|
4410
|
+
const inputs = Array.from({
|
|
4411
|
+
length: fanSize
|
|
4412
|
+
}, (_, i) => ({
|
|
4413
|
+
i
|
|
4414
|
+
}));
|
|
4415
|
+
return settle2 ? ctx.allSettled("child", inputs) : ctx.all("child", inputs);
|
|
4416
|
+
});
|
|
4417
|
+
}
|
|
4418
|
+
}
|
|
4419
|
+
__name(registerFan, "registerFan");
|
|
4420
|
+
function doubled(fanSize, settle2) {
|
|
4421
|
+
const values = Array.from({
|
|
4422
|
+
length: fanSize
|
|
4423
|
+
}, (_, i) => i * 2);
|
|
4424
|
+
return settle2 ? values.map((value) => ({
|
|
4425
|
+
status: "fulfilled",
|
|
4426
|
+
value
|
|
4427
|
+
})) : values;
|
|
4428
|
+
}
|
|
4429
|
+
__name(doubled, "doubled");
|
|
4430
|
+
async function shutdown(all, transports) {
|
|
4431
|
+
await Promise.allSettled(all.map(({ engine }) => engine.drain(2e3)));
|
|
4432
|
+
await Promise.allSettled(transports.map((t) => t.close?.()));
|
|
4433
|
+
}
|
|
4434
|
+
__name(shutdown, "shutdown");
|
|
4435
|
+
function runRunDispatchContract(name, makeHarness, options = {}) {
|
|
4436
|
+
const timeoutMs = options.timeoutMs ?? 3e4;
|
|
4437
|
+
const fanSize = options.fanSize ?? 300;
|
|
4438
|
+
const dedupes = options.dedupesWaitingRuns ?? true;
|
|
4439
|
+
describe3(`Run dispatch contract: ${name}`, () => {
|
|
4440
|
+
async function pairOrSkip(skip) {
|
|
4441
|
+
const harness = await makeHarness();
|
|
4442
|
+
if (typeof harness === "string") {
|
|
4443
|
+
console.warn(`[run-dispatch contract] ${name} skipped: ${harness}`);
|
|
4444
|
+
skip();
|
|
4445
|
+
return void 0;
|
|
4446
|
+
}
|
|
4447
|
+
return harness.pair();
|
|
4448
|
+
}
|
|
4449
|
+
__name(pairOrSkip, "pairOrSkip");
|
|
4450
|
+
for (const { fan, settle: settle2 } of FANS) {
|
|
4451
|
+
it3(`queues a ${fan}'s children on the transport in batches, and the fan completes`, async (ctx) => {
|
|
4452
|
+
const pair = await pairOrSkip(() => ctx.skip());
|
|
4453
|
+
if (!pair) return;
|
|
4454
|
+
const a = recorded(pair[0]);
|
|
4455
|
+
const b = recorded(pair[1]);
|
|
4456
|
+
const store = new InMemoryStateStore4();
|
|
4457
|
+
const all = instances(store, [
|
|
4458
|
+
a.transport,
|
|
4459
|
+
b.transport
|
|
4460
|
+
]);
|
|
4461
|
+
const ran = /* @__PURE__ */ new Map();
|
|
4462
|
+
registerFan(all, fanSize, ran, settle2);
|
|
4463
|
+
try {
|
|
4464
|
+
for (const { engine } of all) expect4(engine.consumeRuns()).toBeTypeOf("function");
|
|
4465
|
+
await all[0]?.engine.start("parent", {}, "p");
|
|
4466
|
+
await until(async () => (await store.getRun("p"))?.status === "completed", timeoutMs);
|
|
4467
|
+
expect4((await store.getRun("p"))?.output).toEqual(doubled(fanSize, settle2));
|
|
4468
|
+
const batches = [
|
|
4469
|
+
...a.batches,
|
|
4470
|
+
...b.batches
|
|
4471
|
+
];
|
|
4472
|
+
const queued = batches.flat().map((r) => r.runId);
|
|
4473
|
+
const children = (await store.listRuns({
|
|
4474
|
+
workflow: "child"
|
|
4475
|
+
})).map((r) => r.id);
|
|
4476
|
+
expect4(children).toHaveLength(fanSize);
|
|
4477
|
+
expect4(new Set(queued)).toEqual(/* @__PURE__ */ new Set([
|
|
4478
|
+
"p",
|
|
4479
|
+
...children
|
|
4480
|
+
]));
|
|
4481
|
+
expect4(Math.max(...batches.map((batch) => batch.length))).toBeLessThanOrEqual(500);
|
|
4482
|
+
expect4(batches.length).toBeLessThanOrEqual(Math.ceil(fanSize / 20));
|
|
4483
|
+
} finally {
|
|
4484
|
+
await shutdown(all, pair);
|
|
4485
|
+
}
|
|
4486
|
+
}, timeoutMs + 1e4);
|
|
4487
|
+
it3(`splits a ${fan}'s children across the instances, each child executed once`, async (ctx) => {
|
|
4488
|
+
const pair = await pairOrSkip(() => ctx.skip());
|
|
4489
|
+
if (!pair) return;
|
|
4490
|
+
const store = new InMemoryStateStore4();
|
|
4491
|
+
const all = instances(store, pair);
|
|
4492
|
+
const ran = /* @__PURE__ */ new Map();
|
|
4493
|
+
registerFan(all, fanSize, ran, settle2);
|
|
4494
|
+
try {
|
|
4495
|
+
for (const { engine } of all) engine.consumeRuns();
|
|
4496
|
+
await all[0]?.engine.start("parent", {}, "p");
|
|
4497
|
+
await until(async () => (await store.getRun("p"))?.status === "completed", timeoutMs);
|
|
4498
|
+
expect4(ran.size).toBe(fanSize);
|
|
4499
|
+
for (const executors of ran.values()) expect4(executors).toHaveLength(1);
|
|
4500
|
+
const byInstance = /* @__PURE__ */ new Map();
|
|
4501
|
+
for (const [executor] of ran.values()) {
|
|
4502
|
+
byInstance.set(executor, (byInstance.get(executor) ?? 0) + 1);
|
|
4503
|
+
}
|
|
4504
|
+
expect4(byInstance.get("instance-0") ?? 0).toBeGreaterThan(0);
|
|
4505
|
+
expect4(byInstance.get("instance-1") ?? 0).toBeGreaterThan(0);
|
|
4506
|
+
} finally {
|
|
4507
|
+
await shutdown(all, pair);
|
|
4508
|
+
}
|
|
4509
|
+
}, timeoutMs + 1e4);
|
|
4510
|
+
}
|
|
4511
|
+
it3("leaves a run whose queueing failed pending, and the poll takes it", async (ctx) => {
|
|
4512
|
+
const pair = await pairOrSkip(() => ctx.skip());
|
|
4513
|
+
if (!pair) return;
|
|
4514
|
+
const a = recorded(pair[0]);
|
|
4515
|
+
const store = new InMemoryStateStore4();
|
|
4516
|
+
const all = instances(store, [
|
|
4517
|
+
a.transport
|
|
4518
|
+
]);
|
|
4519
|
+
let turns = 0;
|
|
4520
|
+
all[0]?.engine.register("wf", "1", async () => {
|
|
4521
|
+
turns += 1;
|
|
4522
|
+
return "done";
|
|
4523
|
+
});
|
|
4524
|
+
const engine = all[0]?.engine;
|
|
4525
|
+
try {
|
|
4526
|
+
engine.consumeRuns();
|
|
4527
|
+
a.failing.on = true;
|
|
4528
|
+
await engine.start("wf", {}, "lost");
|
|
4529
|
+
await engine.start("wf", {}, "kept");
|
|
4530
|
+
await pause(100);
|
|
4531
|
+
expect4((await store.getRun("lost"))?.status).toBe("pending");
|
|
4532
|
+
expect4(turns).toBe(0);
|
|
4533
|
+
a.failing.on = false;
|
|
4534
|
+
await engine.runPending();
|
|
4535
|
+
await until(async () => (await store.getRun("lost"))?.status === "completed", timeoutMs);
|
|
4536
|
+
expect4((await store.getRun("kept"))?.status).toBe("completed");
|
|
4537
|
+
a.failing.on = true;
|
|
4538
|
+
await engine.start("wf", {}, "down");
|
|
4539
|
+
await pause(50);
|
|
4540
|
+
expect4((await store.getRun("down"))?.status).toBe("pending");
|
|
4541
|
+
await engine.runPending();
|
|
4542
|
+
await until(async () => (await store.getRun("down"))?.status === "completed", timeoutMs);
|
|
4543
|
+
expect4(turns).toBe(3);
|
|
4544
|
+
} finally {
|
|
4545
|
+
a.failing.on = false;
|
|
4546
|
+
await shutdown(all, [
|
|
4547
|
+
pair[0],
|
|
4548
|
+
pair[1]
|
|
4549
|
+
]);
|
|
4550
|
+
}
|
|
4551
|
+
}, timeoutMs + 1e4);
|
|
4552
|
+
it3("skips a queued run that is no longer due", async (ctx) => {
|
|
4553
|
+
const pair = await pairOrSkip(() => ctx.skip());
|
|
4554
|
+
if (!pair) return;
|
|
4555
|
+
const store = new InMemoryStateStore4();
|
|
4556
|
+
const [local, consumer] = instances(store, pair);
|
|
4557
|
+
let turns = 0;
|
|
4558
|
+
for (const { engine } of [
|
|
4559
|
+
local,
|
|
4560
|
+
consumer
|
|
4561
|
+
]) {
|
|
4562
|
+
engine.register("done", "1", async () => {
|
|
4563
|
+
turns += 1;
|
|
4564
|
+
return 1;
|
|
4565
|
+
});
|
|
4566
|
+
engine.register("waiting", "1", async (ctx2) => {
|
|
4567
|
+
turns += 1;
|
|
4568
|
+
await ctx2.waitForSignal("never");
|
|
4569
|
+
});
|
|
4570
|
+
}
|
|
4571
|
+
try {
|
|
4572
|
+
await local.engine.start("done", {}, "finished");
|
|
4573
|
+
await local.engine.start("waiting", {}, "parked");
|
|
4574
|
+
await until(async () => (await store.getRun("parked"))?.status === "suspended", timeoutMs);
|
|
4575
|
+
await until(async () => (await store.getRun("finished"))?.status === "completed", timeoutMs);
|
|
4576
|
+
expect4(turns).toBe(2);
|
|
4577
|
+
consumer.engine.consumeRuns();
|
|
4578
|
+
await pair[0].dispatchRuns?.([
|
|
4579
|
+
{
|
|
4580
|
+
runId: "finished",
|
|
4581
|
+
reason: "start",
|
|
4582
|
+
key: "1"
|
|
4583
|
+
},
|
|
4584
|
+
{
|
|
4585
|
+
runId: "parked",
|
|
4586
|
+
reason: "start",
|
|
4587
|
+
key: "1"
|
|
4588
|
+
}
|
|
4589
|
+
]);
|
|
4590
|
+
await pause(1500);
|
|
4591
|
+
expect4(turns).toBe(2);
|
|
4592
|
+
expect4((await store.getRun("parked"))?.status).toBe("suspended");
|
|
4593
|
+
} finally {
|
|
4594
|
+
await shutdown([
|
|
4595
|
+
local,
|
|
4596
|
+
consumer
|
|
4597
|
+
], pair);
|
|
4598
|
+
}
|
|
4599
|
+
}, timeoutMs + 1e4);
|
|
4600
|
+
it3("a wake resumes a parked run, and skips a finished one", async (ctx) => {
|
|
4601
|
+
const pair = await pairOrSkip(() => ctx.skip());
|
|
4602
|
+
if (!pair) return;
|
|
4603
|
+
const store = new InMemoryStateStore4();
|
|
4604
|
+
const [local, consumer] = instances(store, pair);
|
|
4605
|
+
let parkedTurns = 0;
|
|
4606
|
+
let finishedTurns = 0;
|
|
4607
|
+
for (const { engine } of [
|
|
4608
|
+
local,
|
|
4609
|
+
consumer
|
|
4610
|
+
]) {
|
|
4611
|
+
engine.register("done", "1", async () => {
|
|
4612
|
+
finishedTurns += 1;
|
|
4613
|
+
return 1;
|
|
4614
|
+
});
|
|
4615
|
+
engine.register("waiting", "1", async (wctx) => {
|
|
4616
|
+
parkedTurns += 1;
|
|
4617
|
+
await wctx.waitForSignal("never");
|
|
4618
|
+
});
|
|
4619
|
+
}
|
|
4620
|
+
try {
|
|
4621
|
+
await local.engine.start("done", {}, "finished");
|
|
4622
|
+
await local.engine.start("waiting", {}, "parked");
|
|
4623
|
+
await until(async () => (await store.getRun("parked"))?.status === "suspended", timeoutMs);
|
|
4624
|
+
await until(async () => (await store.getRun("finished"))?.status === "completed", timeoutMs);
|
|
4625
|
+
consumer.engine.consumeRuns();
|
|
4626
|
+
await pair[0].dispatchRuns?.([
|
|
4627
|
+
{
|
|
4628
|
+
runId: "finished",
|
|
4629
|
+
reason: "wake"
|
|
4630
|
+
},
|
|
4631
|
+
{
|
|
4632
|
+
runId: "parked",
|
|
4633
|
+
reason: "wake"
|
|
4634
|
+
}
|
|
4635
|
+
]);
|
|
4636
|
+
await until(() => parkedTurns === 2, timeoutMs);
|
|
4637
|
+
await pause(500);
|
|
4638
|
+
expect4(parkedTurns).toBe(2);
|
|
4639
|
+
expect4(finishedTurns).toBe(1);
|
|
4640
|
+
expect4((await store.getRun("parked"))?.status).toBe("suspended");
|
|
4641
|
+
} finally {
|
|
4642
|
+
await shutdown([
|
|
4643
|
+
local,
|
|
4644
|
+
consumer
|
|
4645
|
+
], pair);
|
|
4646
|
+
}
|
|
4647
|
+
}, timeoutMs + 1e4);
|
|
4648
|
+
it3("a signal's wake goes through the queue, and signal resolves once it is queued", async (ctx) => {
|
|
4649
|
+
const pair = await pairOrSkip(() => ctx.skip());
|
|
4650
|
+
if (!pair) return;
|
|
4651
|
+
const store = new InMemoryStateStore4();
|
|
4652
|
+
const a = recorded(pair[0]);
|
|
4653
|
+
const all = instances(store, [
|
|
4654
|
+
a.transport,
|
|
4655
|
+
pair[1]
|
|
4656
|
+
]);
|
|
4657
|
+
const resumedOn = [];
|
|
4658
|
+
for (const { name: name2, engine } of all) {
|
|
4659
|
+
engine.register("waits", "1", async (wctx) => {
|
|
4660
|
+
const got = await wctx.waitForSignal("go");
|
|
4661
|
+
return wctx.localStep("after", async () => {
|
|
4662
|
+
resumedOn.push(name2);
|
|
4663
|
+
return got;
|
|
4664
|
+
});
|
|
4665
|
+
});
|
|
4666
|
+
engine.consumeRuns();
|
|
4667
|
+
}
|
|
4668
|
+
try {
|
|
4669
|
+
await all[0]?.engine.start("waits", {}, "w");
|
|
4670
|
+
await until(async () => (await store.getRun("w"))?.status === "suspended", timeoutMs);
|
|
4671
|
+
a.batches.length = 0;
|
|
4672
|
+
const result = await all[0]?.engine.signal("go", "payload");
|
|
4673
|
+
expect4(result?.status).toBe("suspended");
|
|
4674
|
+
await until(async () => (await store.getRun("w"))?.status === "completed", timeoutMs);
|
|
4675
|
+
expect4(a.batches.flat()).toEqual([
|
|
4676
|
+
{
|
|
4677
|
+
runId: "w",
|
|
4678
|
+
reason: "wake"
|
|
4679
|
+
}
|
|
4680
|
+
]);
|
|
4681
|
+
expect4((await store.getRun("w"))?.output).toBe("payload");
|
|
4682
|
+
expect4(resumedOn).toHaveLength(1);
|
|
4683
|
+
} finally {
|
|
4684
|
+
await shutdown(all, pair);
|
|
4685
|
+
}
|
|
4686
|
+
}, timeoutMs + 1e4);
|
|
4687
|
+
it3("queues due timers on the transport instead of resuming them inline in the poll", async (ctx) => {
|
|
4688
|
+
const pair = await pairOrSkip(() => ctx.skip());
|
|
4689
|
+
if (!pair) return;
|
|
4690
|
+
const a = recorded(pair[0]);
|
|
4691
|
+
let now = Date.now();
|
|
4692
|
+
const store = new InMemoryStateStore4();
|
|
4693
|
+
const all = instances(store, [
|
|
4694
|
+
a.transport,
|
|
4695
|
+
pair[1]
|
|
4696
|
+
], () => now);
|
|
4697
|
+
const n = 40;
|
|
4698
|
+
const woke = /* @__PURE__ */ new Map();
|
|
4699
|
+
for (const { name: name2, engine } of all) {
|
|
4700
|
+
engine.register("sleeper", "1", async (ctx2, input) => {
|
|
4701
|
+
const { i } = input;
|
|
4702
|
+
await ctx2.sleep(1e3);
|
|
4703
|
+
return ctx2.localStep("after", async () => {
|
|
4704
|
+
woke.set(i, [
|
|
4705
|
+
...woke.get(i) ?? [],
|
|
4706
|
+
name2
|
|
4707
|
+
]);
|
|
4708
|
+
await pause(15);
|
|
4709
|
+
return i;
|
|
4710
|
+
});
|
|
4711
|
+
});
|
|
4712
|
+
}
|
|
4713
|
+
try {
|
|
4714
|
+
for (const { engine } of all) engine.consumeRuns();
|
|
4715
|
+
for (let i = 0; i < n; i += 1) await all[0]?.engine.start("sleeper", {
|
|
4716
|
+
i
|
|
4717
|
+
}, `s${i}`);
|
|
4718
|
+
await until(async () => {
|
|
4719
|
+
const runs = await store.listRuns({
|
|
4720
|
+
workflow: "sleeper",
|
|
4721
|
+
status: "suspended"
|
|
4722
|
+
});
|
|
4723
|
+
return runs.length === n;
|
|
4724
|
+
}, timeoutMs);
|
|
4725
|
+
a.batches.length = 0;
|
|
4726
|
+
now += 2e3;
|
|
4727
|
+
const results = await all[0]?.engine.resumeDueTimers(now);
|
|
4728
|
+
expect4(results).toHaveLength(n);
|
|
4729
|
+
expect4(results?.every((r) => r.status === "suspended")).toBe(true);
|
|
4730
|
+
expect4(new Set(a.batches.flat().map((r) => r.runId)).size).toBe(n);
|
|
4731
|
+
await until(async () => {
|
|
4732
|
+
const runs = await store.listRuns({
|
|
4733
|
+
workflow: "sleeper",
|
|
4734
|
+
status: "completed"
|
|
4735
|
+
});
|
|
4736
|
+
if (runs.length === n) return true;
|
|
4737
|
+
await all[0]?.engine.resumeDueTimers(now);
|
|
4738
|
+
await pause(50);
|
|
4739
|
+
return false;
|
|
4740
|
+
}, timeoutMs);
|
|
4741
|
+
for (const executors of woke.values()) expect4(executors).toHaveLength(1);
|
|
4742
|
+
} finally {
|
|
4743
|
+
await shutdown(all, pair);
|
|
4744
|
+
}
|
|
4745
|
+
}, timeoutMs + 1e4);
|
|
4746
|
+
it3("hands a consumer at most `concurrency` runs at once", async (ctx) => {
|
|
4747
|
+
const pair = await pairOrSkip(() => ctx.skip());
|
|
4748
|
+
if (!pair) return;
|
|
4749
|
+
const [producer, consumerTransport] = pair;
|
|
4750
|
+
let active = 0;
|
|
4751
|
+
let peak = 0;
|
|
4752
|
+
const handled = [];
|
|
4753
|
+
let release;
|
|
4754
|
+
const gate = new Promise((resolve) => {
|
|
4755
|
+
release = resolve;
|
|
4756
|
+
});
|
|
4757
|
+
const opts = {
|
|
4758
|
+
concurrency: 3
|
|
4759
|
+
};
|
|
4760
|
+
const stop = consumerTransport.onDispatchedRun?.(async ({ runId }) => {
|
|
4761
|
+
active += 1;
|
|
4762
|
+
peak = Math.max(peak, active);
|
|
4763
|
+
await gate;
|
|
4764
|
+
handled.push(runId);
|
|
4765
|
+
active -= 1;
|
|
4766
|
+
}, opts);
|
|
4767
|
+
try {
|
|
4768
|
+
await producer.dispatchRuns?.(Array.from({
|
|
4769
|
+
length: 8
|
|
4770
|
+
}, (_, i) => ({
|
|
4771
|
+
runId: `c${i}`
|
|
4772
|
+
})));
|
|
4773
|
+
await until(() => active === 3, timeoutMs);
|
|
4774
|
+
await pause(300);
|
|
4775
|
+
expect4(peak).toBe(3);
|
|
4776
|
+
release();
|
|
4777
|
+
await until(() => handled.length === 8, timeoutMs);
|
|
4778
|
+
expect4(new Set(handled).size).toBe(8);
|
|
4779
|
+
expect4(peak).toBe(3);
|
|
4780
|
+
} finally {
|
|
4781
|
+
release();
|
|
4782
|
+
await stop?.();
|
|
4783
|
+
await Promise.allSettled(pair.map((t) => t.close?.()));
|
|
4784
|
+
}
|
|
4785
|
+
}, timeoutMs + 1e4);
|
|
4786
|
+
it3.runIf(!dedupes)("runs one turn of a run queued twice", async (ctx) => {
|
|
4787
|
+
const pair = await pairOrSkip(() => ctx.skip());
|
|
4788
|
+
if (!pair) return;
|
|
4789
|
+
const store = new InMemoryStateStore4();
|
|
4790
|
+
const all = instances(store, pair);
|
|
4791
|
+
let turns = 0;
|
|
4792
|
+
for (const { engine } of all) {
|
|
4793
|
+
engine.register("wf", "1", async (wctx) => {
|
|
4794
|
+
turns += 1;
|
|
4795
|
+
await wctx.waitForSignal("never");
|
|
4796
|
+
});
|
|
4797
|
+
}
|
|
4798
|
+
try {
|
|
4799
|
+
await store.createRun({
|
|
4800
|
+
id: "twice",
|
|
4801
|
+
workflow: "wf",
|
|
4802
|
+
workflowVersion: "1",
|
|
4803
|
+
status: "pending",
|
|
4804
|
+
input: {},
|
|
4805
|
+
createdAt: /* @__PURE__ */ new Date(),
|
|
4806
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
4807
|
+
});
|
|
4808
|
+
await pair[0].dispatchRuns?.([
|
|
4809
|
+
{
|
|
4810
|
+
runId: "twice",
|
|
4811
|
+
reason: "start",
|
|
4812
|
+
key: "1"
|
|
4813
|
+
}
|
|
4814
|
+
]);
|
|
4815
|
+
await pair[0].dispatchRuns?.([
|
|
4816
|
+
{
|
|
4817
|
+
runId: "twice",
|
|
4818
|
+
reason: "start",
|
|
4819
|
+
key: "2"
|
|
4820
|
+
}
|
|
4821
|
+
]);
|
|
4822
|
+
for (const { engine } of all) engine.consumeRuns();
|
|
4823
|
+
await until(async () => (await store.getRun("twice"))?.status === "suspended", timeoutMs);
|
|
4824
|
+
await pause(1500);
|
|
4825
|
+
expect4(turns).toBe(1);
|
|
4826
|
+
} finally {
|
|
4827
|
+
await shutdown(all, pair);
|
|
4828
|
+
}
|
|
4829
|
+
}, timeoutMs + 1e4);
|
|
4830
|
+
it3.runIf(dedupes)("queues a run once while it waits, however often it is dispatched", async (ctx) => {
|
|
4831
|
+
const pair = await pairOrSkip(() => ctx.skip());
|
|
4832
|
+
if (!pair) return;
|
|
4833
|
+
const [producer, consumerTransport] = pair;
|
|
4834
|
+
const handled = [];
|
|
4835
|
+
try {
|
|
4836
|
+
const once = {
|
|
4837
|
+
runId: "once",
|
|
4838
|
+
reason: "start",
|
|
4839
|
+
key: "1"
|
|
4840
|
+
};
|
|
4841
|
+
await producer.dispatchRuns?.([
|
|
4842
|
+
once
|
|
4843
|
+
]);
|
|
4844
|
+
await producer.dispatchRuns?.([
|
|
4845
|
+
once,
|
|
4846
|
+
once
|
|
4847
|
+
]);
|
|
4848
|
+
const stop = consumerTransport.onDispatchedRun?.(async ({ runId }) => {
|
|
4849
|
+
handled.push(runId);
|
|
4850
|
+
}, {
|
|
4851
|
+
concurrency: 4
|
|
4852
|
+
});
|
|
4853
|
+
await until(() => handled.length >= 1, timeoutMs);
|
|
4854
|
+
await pause(500);
|
|
4855
|
+
await stop?.();
|
|
4856
|
+
expect4(handled).toEqual([
|
|
4857
|
+
"once"
|
|
4858
|
+
]);
|
|
4859
|
+
} finally {
|
|
4860
|
+
await Promise.allSettled(pair.map((t) => t.close?.()));
|
|
4861
|
+
}
|
|
4862
|
+
}, timeoutMs + 1e4);
|
|
4863
|
+
for (const [what, again] of [
|
|
4864
|
+
[
|
|
4865
|
+
"a start for a new pending state",
|
|
4866
|
+
{
|
|
4867
|
+
reason: "start",
|
|
4868
|
+
key: "2"
|
|
4869
|
+
}
|
|
4870
|
+
],
|
|
4871
|
+
[
|
|
4872
|
+
"a wake",
|
|
4873
|
+
{
|
|
4874
|
+
reason: "wake"
|
|
4875
|
+
}
|
|
4876
|
+
]
|
|
4877
|
+
]) {
|
|
4878
|
+
it3(`hands a run out again when ${what} lands while its entry is being handled`, async (ctx) => {
|
|
4879
|
+
const pair = await pairOrSkip(() => ctx.skip());
|
|
4880
|
+
if (!pair) return;
|
|
4881
|
+
const [producer, consumerTransport] = pair;
|
|
4882
|
+
const handled = [];
|
|
4883
|
+
let release;
|
|
4884
|
+
const gate = new Promise((resolve) => {
|
|
4885
|
+
release = resolve;
|
|
4886
|
+
});
|
|
4887
|
+
let taken = 0;
|
|
4888
|
+
const stop = consumerTransport.onDispatchedRun?.(async ({ runId }) => {
|
|
4889
|
+
taken += 1;
|
|
4890
|
+
if (taken === 1) await gate;
|
|
4891
|
+
handled.push(runId);
|
|
4892
|
+
}, {
|
|
4893
|
+
concurrency: 2
|
|
4894
|
+
});
|
|
4895
|
+
try {
|
|
4896
|
+
await producer.dispatchRuns?.([
|
|
4897
|
+
{
|
|
4898
|
+
runId: "busy",
|
|
4899
|
+
reason: "start",
|
|
4900
|
+
key: "1"
|
|
4901
|
+
}
|
|
4902
|
+
]);
|
|
4903
|
+
await until(() => taken === 1, timeoutMs);
|
|
4904
|
+
await producer.dispatchRuns?.([
|
|
4905
|
+
{
|
|
4906
|
+
runId: "busy",
|
|
4907
|
+
...again
|
|
4908
|
+
}
|
|
4909
|
+
]);
|
|
4910
|
+
await pause(200);
|
|
4911
|
+
release();
|
|
4912
|
+
await until(() => handled.length >= 2, timeoutMs);
|
|
4913
|
+
expect4(handled.slice(0, 2)).toEqual([
|
|
4914
|
+
"busy",
|
|
4915
|
+
"busy"
|
|
4916
|
+
]);
|
|
4917
|
+
} finally {
|
|
4918
|
+
release();
|
|
4919
|
+
await stop?.();
|
|
4920
|
+
await Promise.allSettled(pair.map((t) => t.close?.()));
|
|
4921
|
+
}
|
|
4922
|
+
}, timeoutMs + 1e4);
|
|
4923
|
+
}
|
|
4924
|
+
it3("stops taking runs on stop, and resolves once the runs in flight settled", async (ctx) => {
|
|
4925
|
+
const pair = await pairOrSkip(() => ctx.skip());
|
|
4926
|
+
if (!pair) return;
|
|
4927
|
+
const [first, second] = pair;
|
|
4928
|
+
const handledFirst = [];
|
|
4929
|
+
let active = 0;
|
|
4930
|
+
let release;
|
|
4931
|
+
const gate = new Promise((resolve) => {
|
|
4932
|
+
release = resolve;
|
|
4933
|
+
});
|
|
4934
|
+
const stopFirst = first.onDispatchedRun?.(async ({ runId }) => {
|
|
4935
|
+
active += 1;
|
|
4936
|
+
await gate;
|
|
4937
|
+
handledFirst.push(runId);
|
|
4938
|
+
active -= 1;
|
|
4939
|
+
}, {
|
|
4940
|
+
concurrency: 2
|
|
4941
|
+
});
|
|
4942
|
+
try {
|
|
4943
|
+
await first.dispatchRuns?.([
|
|
4944
|
+
{
|
|
4945
|
+
runId: "in-flight-1"
|
|
4946
|
+
},
|
|
4947
|
+
{
|
|
4948
|
+
runId: "in-flight-2"
|
|
4949
|
+
}
|
|
4950
|
+
]);
|
|
4951
|
+
await until(() => active === 2, timeoutMs);
|
|
4952
|
+
let stopped = false;
|
|
4953
|
+
const stopping = stopFirst?.().then(() => {
|
|
4954
|
+
stopped = true;
|
|
4955
|
+
});
|
|
4956
|
+
await pause(200);
|
|
4957
|
+
expect4(stopped).toBe(false);
|
|
4958
|
+
await first.dispatchRuns?.([
|
|
4959
|
+
{
|
|
4960
|
+
runId: "after-1"
|
|
4961
|
+
},
|
|
4962
|
+
{
|
|
4963
|
+
runId: "after-2"
|
|
4964
|
+
}
|
|
4965
|
+
]);
|
|
4966
|
+
release();
|
|
4967
|
+
await stopping;
|
|
4968
|
+
expect4(new Set(handledFirst)).toEqual(/* @__PURE__ */ new Set([
|
|
4969
|
+
"in-flight-1",
|
|
4970
|
+
"in-flight-2"
|
|
4971
|
+
]));
|
|
4972
|
+
await pause(300);
|
|
4973
|
+
expect4(handledFirst).toHaveLength(2);
|
|
4974
|
+
const handledSecond = [];
|
|
4975
|
+
const stopSecond = second.onDispatchedRun?.(async ({ runId }) => {
|
|
4976
|
+
handledSecond.push(runId);
|
|
4977
|
+
}, {
|
|
4978
|
+
concurrency: 2
|
|
4979
|
+
});
|
|
4980
|
+
await until(() => handledSecond.length === 2, timeoutMs);
|
|
4981
|
+
expect4(new Set(handledSecond)).toEqual(/* @__PURE__ */ new Set([
|
|
4982
|
+
"after-1",
|
|
4983
|
+
"after-2"
|
|
4984
|
+
]));
|
|
4985
|
+
await stopSecond?.();
|
|
4986
|
+
} finally {
|
|
4987
|
+
release();
|
|
4988
|
+
await Promise.allSettled(pair.map((t) => t.close?.()));
|
|
4989
|
+
}
|
|
4990
|
+
}, timeoutMs + 1e4);
|
|
4991
|
+
it3("drain waits for a run taken off the queue, and the draining instance takes no more", async (ctx) => {
|
|
4992
|
+
const pair = await pairOrSkip(() => ctx.skip());
|
|
4993
|
+
if (!pair) return;
|
|
4994
|
+
const store = new InMemoryStateStore4();
|
|
4995
|
+
const all = instances(store, pair);
|
|
4996
|
+
let release;
|
|
4997
|
+
const gate = new Promise((resolve) => {
|
|
4998
|
+
release = resolve;
|
|
4999
|
+
});
|
|
5000
|
+
const ranOn = /* @__PURE__ */ new Map();
|
|
5001
|
+
for (const { name: name2, engine } of all) {
|
|
5002
|
+
engine.register("slow", "1", async (ctx2, input) => ctx2.localStep("work", async () => {
|
|
5003
|
+
const { id } = input;
|
|
5004
|
+
ranOn.set(id, name2);
|
|
5005
|
+
if (id === "held") await gate;
|
|
5006
|
+
return id;
|
|
5007
|
+
}));
|
|
5008
|
+
}
|
|
5009
|
+
const [draining, other] = all;
|
|
5010
|
+
try {
|
|
5011
|
+
draining.engine.consumeRuns();
|
|
5012
|
+
await draining.engine.start("slow", {
|
|
5013
|
+
id: "held"
|
|
5014
|
+
}, "held");
|
|
5015
|
+
await until(() => ranOn.has("held"), timeoutMs);
|
|
5016
|
+
let drained = false;
|
|
5017
|
+
const drain = draining.engine.drain(timeoutMs).then(() => {
|
|
5018
|
+
drained = true;
|
|
5019
|
+
});
|
|
5020
|
+
await pause(200);
|
|
5021
|
+
expect4(drained).toBe(false);
|
|
5022
|
+
await draining.engine.start("slow", {
|
|
5023
|
+
id: "later"
|
|
5024
|
+
}, "later");
|
|
5025
|
+
await pause(200);
|
|
5026
|
+
expect4(ranOn.has("later")).toBe(false);
|
|
5027
|
+
release();
|
|
5028
|
+
await drain;
|
|
5029
|
+
expect4((await store.getRun("held"))?.status).toBe("completed");
|
|
5030
|
+
other.engine.consumeRuns();
|
|
5031
|
+
await until(async () => (await store.getRun("later"))?.status === "completed", timeoutMs);
|
|
5032
|
+
expect4(ranOn.get("later")).toBe("instance-1");
|
|
5033
|
+
} finally {
|
|
5034
|
+
release();
|
|
5035
|
+
await shutdown(all, pair);
|
|
5036
|
+
}
|
|
5037
|
+
}, timeoutMs + 1e4);
|
|
5038
|
+
});
|
|
5039
|
+
}
|
|
5040
|
+
__name(runRunDispatchContract, "runRunDispatchContract");
|
|
4239
5041
|
export {
|
|
4240
5042
|
DURABLE_CANONICAL_COLUMNS,
|
|
4241
5043
|
MutableClock,
|
|
5044
|
+
RUN_HISTORY_FIXTURE_VERSION,
|
|
4242
5045
|
StateStoreUnavailableError,
|
|
4243
5046
|
aggregationFixture,
|
|
4244
5047
|
assertDurableColumns,
|
|
@@ -4248,13 +5051,17 @@ export {
|
|
|
4248
5051
|
assertStepAttempts,
|
|
4249
5052
|
assertStepsRan,
|
|
4250
5053
|
assertTransportConformance,
|
|
5054
|
+
captureHistory,
|
|
4251
5055
|
createTestEngine,
|
|
4252
5056
|
failOnce,
|
|
4253
5057
|
failTimes,
|
|
5058
|
+
parseRunHistory,
|
|
4254
5059
|
preserveColumnExpectation,
|
|
4255
5060
|
recordedSteps,
|
|
4256
5061
|
registerCheckpointAggregationCases,
|
|
4257
5062
|
runAdmissionBackendContract,
|
|
4258
|
-
|
|
5063
|
+
runRunDispatchContract,
|
|
5064
|
+
runStateStoreContract,
|
|
5065
|
+
serializeRunHistory
|
|
4259
5066
|
};
|
|
4260
5067
|
//# sourceMappingURL=index.js.map
|