@rikalabs/akter 0.1.0-alpha.0 → 0.1.0-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +46 -3
- package/README.md +2 -2
- package/dist/actor/definition.d.ts +13 -9
- package/dist/actor/descriptor.d.ts +2 -2
- package/dist/actor/descriptor.js +4 -4
- package/dist/client/calls.js +1 -1
- package/dist/client/index.d.ts +1 -1
- package/dist/client/index.js +1 -1
- package/dist/errors/actor.d.ts +12 -42
- package/dist/errors/actor.js +13 -42
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/members/payload.d.ts +1 -1
- package/dist/protocol/inspection.d.ts +221 -1
- package/dist/protocol/inspection.js +151 -2
- package/dist/protocol/wire.d.ts +6 -49
- package/dist/protocol/wire.js +1 -10
- package/dist/runtime/actors.d.ts +3 -0
- package/dist/runtime/adoption/backfill.js +1 -1
- package/dist/runtime/adoption/enforce.js +2 -2
- package/dist/runtime/adoption/observe.js +1 -1
- package/dist/runtime/adoption/plan.d.ts +1 -1
- package/dist/runtime/adoption/startup.js +1 -1
- package/dist/runtime/adoption/status.d.ts +1 -1
- package/dist/runtime/app.d.ts +65 -0
- package/dist/runtime/app.js +38 -0
- package/dist/runtime/connections/holder.d.ts +11 -1
- package/dist/runtime/connections/holder.js +8 -4
- package/dist/runtime/connections/streams.d.ts +4 -4
- package/dist/runtime/content/store.js +15 -11
- package/dist/runtime/database/bounded.d.ts +4 -18
- package/dist/runtime/database/bounded.js +36 -22
- package/dist/runtime/database/coordination.d.ts +12 -3
- package/dist/runtime/database/coordination.js +29 -36
- package/dist/runtime/database/migrations.d.ts +61 -29
- package/dist/runtime/database/migrations.js +375 -139
- package/dist/runtime/database/replica.d.ts +4 -1
- package/dist/runtime/database/replica.js +4 -1
- package/dist/runtime/database/schema.d.ts +13 -0
- package/dist/runtime/database/schema.js +13 -0
- package/dist/runtime/database/shards.d.ts +31 -13
- package/dist/runtime/database/shards.js +38 -26
- package/dist/runtime/entity/register.d.ts +8 -1
- package/dist/runtime/entity/register.js +58 -7
- package/dist/runtime/fleet/checks.js +4 -4
- package/dist/runtime/fleet/maintainer.d.ts +1 -1
- package/dist/runtime/fleet/maintainer.js +1 -1
- package/dist/runtime/index.d.ts +2 -0
- package/dist/runtime/index.js +1 -0
- package/dist/runtime/inspector/layer.d.ts +20 -0
- package/dist/runtime/inspector/layer.js +224 -5
- package/dist/runtime/inspector/queries.d.ts +54 -8
- package/dist/runtime/inspector/queries.js +84 -15
- package/dist/runtime/layer.d.ts +48 -39
- package/dist/runtime/layer.js +124 -56
- package/dist/runtime/operators/audit.d.ts +3 -0
- package/dist/runtime/operators/audit.js +6 -2
- package/dist/runtime/operators/repair.js +30 -10
- package/dist/runtime/operators/seed.js +4 -2
- package/dist/runtime/payloads/versions.d.ts +4 -4
- package/dist/runtime/payloads/versions.js +24 -15
- package/dist/runtime/queries.js +4 -2
- package/dist/runtime/registration.d.ts +5 -2
- package/dist/runtime/registration.js +3 -3
- package/dist/runtime/runner.d.ts +12 -0
- package/dist/runtime/runner.js +12 -0
- package/dist/runtime/storage/codec.d.ts +19 -7
- package/dist/runtime/storage/codec.js +23 -7
- package/dist/runtime/storage/placements.d.ts +6 -1
- package/dist/runtime/storage/placements.js +98 -6
- package/dist/runtime/storage/retention.d.ts +6 -0
- package/dist/runtime/storage/retention.js +25 -13
- package/dist/runtime/subscriptions/relay.js +47 -44
- package/dist/runtime/subscriptions/storage.d.ts +3 -2
- package/dist/runtime/subscriptions/storage.js +9 -6
- package/dist/runtime/telemetry/live.d.ts +255 -0
- package/dist/runtime/telemetry/live.js +540 -0
- package/dist/runtime/telemetry/metrics.d.ts +2 -0
- package/dist/runtime/telemetry/metrics.js +4 -1
- package/dist/runtime/telemetry/sampler.js +33 -13
- package/dist/runtime/telemetry/spans.d.ts +1 -1
- package/dist/runtime/telemetry/spans.js +1 -1
- package/dist/runtime/topology/locks.d.ts +6 -3
- package/dist/runtime/topology/locks.js +27 -4
- package/dist/runtime/turn/execute.d.ts +2 -0
- package/dist/runtime/turn/execute.js +5 -2
- package/dist/runtime/turn/pipeline.d.ts +5 -12
- package/dist/runtime/turn/pipeline.js +11 -11
- package/dist/runtime/turn/relay.d.ts +13 -3
- package/dist/runtime/turn/relay.js +27 -14
- package/dist/runtime/turn/rows.js +4 -4
- package/dist/runtime/workflows/compatibility.d.ts +1 -1
- package/dist/runtime/workflows/compatibility.js +85 -31
- package/dist/runtime/workflows/engine.js +20 -17
- package/dist/serve/layer.js +3 -2
- package/dist/tables/owned.d.ts +1 -1
- package/dist/tables/owned.js +1 -1
- package/dist/testing/actor-test.d.ts +2 -24
- package/dist/testing/actor-test.js +1 -27
- package/dist/testing/index.d.ts +0 -12
- package/dist/testing/index.js +0 -6
- package/package.json +1 -1
- package/dist/runtime/database/neki/migrations.d.ts +0 -31
- package/dist/runtime/database/neki/migrations.js +0 -276
- package/dist/runtime/database/neki/session.d.ts +0 -23
- package/dist/runtime/database/neki/session.js +0 -31
- package/dist/testing/cluster.d.ts +0 -93
- package/dist/testing/cluster.js +0 -296
- package/dist/testing/conformance/access.d.ts +0 -5
- package/dist/testing/conformance/access.js +0 -531
- package/dist/testing/conformance/admission.d.ts +0 -11
- package/dist/testing/conformance/admission.js +0 -182
- package/dist/testing/conformance/adoption.d.ts +0 -2
- package/dist/testing/conformance/adoption.js +0 -1019
- package/dist/testing/conformance/assertions.d.ts +0 -202
- package/dist/testing/conformance/assertions.js +0 -849
- package/dist/testing/conformance/batches.d.ts +0 -41
- package/dist/testing/conformance/batches.js +0 -388
- package/dist/testing/conformance/blobs.d.ts +0 -14
- package/dist/testing/conformance/blobs.js +0 -405
- package/dist/testing/conformance/capacity.d.ts +0 -8
- package/dist/testing/conformance/capacity.js +0 -433
- package/dist/testing/conformance/client.d.ts +0 -5
- package/dist/testing/conformance/client.js +0 -719
- package/dist/testing/conformance/cold-serve.d.ts +0 -16
- package/dist/testing/conformance/cold-serve.js +0 -254
- package/dist/testing/conformance/connections/actors.d.ts +0 -1143
- package/dist/testing/conformance/connections/actors.js +0 -180
- package/dist/testing/conformance/connections/harness.d.ts +0 -1754
- package/dist/testing/conformance/connections/harness.js +0 -142
- package/dist/testing/conformance/connections/holders.d.ts +0 -4
- package/dist/testing/conformance/connections/holders.js +0 -165
- package/dist/testing/conformance/connections/lifecycle.d.ts +0 -4
- package/dist/testing/conformance/connections/lifecycle.js +0 -113
- package/dist/testing/conformance/connections/limits.d.ts +0 -4
- package/dist/testing/conformance/connections/limits.js +0 -127
- package/dist/testing/conformance/connections/reauthorization.d.ts +0 -4
- package/dist/testing/conformance/connections/reauthorization.js +0 -170
- package/dist/testing/conformance/connections/resync.d.ts +0 -4
- package/dist/testing/conformance/connections/resync.js +0 -360
- package/dist/testing/conformance/connections/takeover.d.ts +0 -4
- package/dist/testing/conformance/connections/takeover.js +0 -287
- package/dist/testing/conformance/connections.d.ts +0 -4
- package/dist/testing/conformance/connections.js +0 -15
- package/dist/testing/conformance/content-blobs.d.ts +0 -16
- package/dist/testing/conformance/content-blobs.js +0 -534
- package/dist/testing/conformance/counter.d.ts +0 -16
- package/dist/testing/conformance/counter.js +0 -813
- package/dist/testing/conformance/cron.d.ts +0 -5
- package/dist/testing/conformance/cron.js +0 -665
- package/dist/testing/conformance/drain.d.ts +0 -26
- package/dist/testing/conformance/drain.js +0 -540
- package/dist/testing/conformance/effect-control.d.ts +0 -38
- package/dist/testing/conformance/effect-control.js +0 -1059
- package/dist/testing/conformance/effects.d.ts +0 -23
- package/dist/testing/conformance/effects.js +0 -432
- package/dist/testing/conformance/events.d.ts +0 -28
- package/dist/testing/conformance/events.js +0 -335
- package/dist/testing/conformance/export.d.ts +0 -2
- package/dist/testing/conformance/export.js +0 -368
- package/dist/testing/conformance/fleet.d.ts +0 -444
- package/dist/testing/conformance/fleet.js +0 -1009
- package/dist/testing/conformance/groups.d.ts +0 -7
- package/dist/testing/conformance/groups.js +0 -353
- package/dist/testing/conformance/heap.d.ts +0 -3
- package/dist/testing/conformance/heap.js +0 -149
- package/dist/testing/conformance/http.d.ts +0 -2809
- package/dist/testing/conformance/http.js +0 -1234
- package/dist/testing/conformance/inspection-views.d.ts +0 -8
- package/dist/testing/conformance/inspection-views.js +0 -272
- package/dist/testing/conformance/inspector.d.ts +0 -7
- package/dist/testing/conformance/inspector.js +0 -650
- package/dist/testing/conformance/mint.d.ts +0 -58
- package/dist/testing/conformance/mint.js +0 -692
- package/dist/testing/conformance/multi-runner.d.ts +0 -8
- package/dist/testing/conformance/multi-runner.js +0 -403
- package/dist/testing/conformance/neki/plan.d.ts +0 -19
- package/dist/testing/conformance/neki/plan.js +0 -30
- package/dist/testing/conformance/observability.d.ts +0 -3
- package/dist/testing/conformance/observability.js +0 -343
- package/dist/testing/conformance/offline.d.ts +0 -4
- package/dist/testing/conformance/offline.js +0 -474
- package/dist/testing/conformance/operator-harness.d.ts +0 -57
- package/dist/testing/conformance/operator-harness.js +0 -78
- package/dist/testing/conformance/operator.d.ts +0 -3
- package/dist/testing/conformance/operator.js +0 -379
- package/dist/testing/conformance/outbox-cross-shard.d.ts +0 -14
- package/dist/testing/conformance/outbox-cross-shard.js +0 -160
- package/dist/testing/conformance/outbox.d.ts +0 -36
- package/dist/testing/conformance/outbox.js +0 -437
- package/dist/testing/conformance/payload-migrations.d.ts +0 -3
- package/dist/testing/conformance/payload-migrations.js +0 -612
- package/dist/testing/conformance/pipeline.d.ts +0 -3
- package/dist/testing/conformance/pipeline.js +0 -1040
- package/dist/testing/conformance/placement.d.ts +0 -22
- package/dist/testing/conformance/placement.js +0 -539
- package/dist/testing/conformance/platform.d.ts +0 -40
- package/dist/testing/conformance/platform.js +0 -101
- package/dist/testing/conformance/progress.d.ts +0 -39
- package/dist/testing/conformance/progress.js +0 -666
- package/dist/testing/conformance/properties.d.ts +0 -6
- package/dist/testing/conformance/properties.js +0 -295
- package/dist/testing/conformance/protocols.d.ts +0 -4
- package/dist/testing/conformance/protocols.js +0 -618
- package/dist/testing/conformance/provider/advertise.d.ts +0 -21
- package/dist/testing/conformance/provider/advertise.js +0 -44
- package/dist/testing/conformance/read-your-writes.d.ts +0 -20
- package/dist/testing/conformance/read-your-writes.js +0 -192
- package/dist/testing/conformance/reducers.d.ts +0 -8
- package/dist/testing/conformance/reducers.js +0 -258
- package/dist/testing/conformance/relay.d.ts +0 -42
- package/dist/testing/conformance/relay.js +0 -1115
- package/dist/testing/conformance/restore.d.ts +0 -43
- package/dist/testing/conformance/restore.js +0 -341
- package/dist/testing/conformance/retention.d.ts +0 -14
- package/dist/testing/conformance/retention.js +0 -463
- package/dist/testing/conformance/rls.d.ts +0 -24
- package/dist/testing/conformance/rls.js +0 -576
- package/dist/testing/conformance/simulation.d.ts +0 -31
- package/dist/testing/conformance/simulation.js +0 -167
- package/dist/testing/conformance/single-shard.d.ts +0 -2
- package/dist/testing/conformance/single-shard.js +0 -125
- package/dist/testing/conformance/singleton.d.ts +0 -3
- package/dist/testing/conformance/singleton.js +0 -293
- package/dist/testing/conformance/statements.d.ts +0 -60
- package/dist/testing/conformance/statements.js +0 -108
- package/dist/testing/conformance/streams.d.ts +0 -7
- package/dist/testing/conformance/streams.js +0 -280
- package/dist/testing/conformance/subscriptions/access.d.ts +0 -4
- package/dist/testing/conformance/subscriptions/access.js +0 -98
- package/dist/testing/conformance/subscriptions/actors.d.ts +0 -4910
- package/dist/testing/conformance/subscriptions/actors.js +0 -372
- package/dist/testing/conformance/subscriptions/cluster.d.ts +0 -4
- package/dist/testing/conformance/subscriptions/cluster.js +0 -89
- package/dist/testing/conformance/subscriptions/delivery.d.ts +0 -4
- package/dist/testing/conformance/subscriptions/delivery.js +0 -253
- package/dist/testing/conformance/subscriptions/deploys.d.ts +0 -4
- package/dist/testing/conformance/subscriptions/deploys.js +0 -162
- package/dist/testing/conformance/subscriptions/epochs.d.ts +0 -4
- package/dist/testing/conformance/subscriptions/epochs.js +0 -169
- package/dist/testing/conformance/subscriptions/failures.d.ts +0 -4
- package/dist/testing/conformance/subscriptions/failures.js +0 -108
- package/dist/testing/conformance/subscriptions/feeds.d.ts +0 -4
- package/dist/testing/conformance/subscriptions/feeds.js +0 -218
- package/dist/testing/conformance/subscriptions/harness.d.ts +0 -124
- package/dist/testing/conformance/subscriptions/harness.js +0 -143
- package/dist/testing/conformance/subscriptions/operator.d.ts +0 -4
- package/dist/testing/conformance/subscriptions/operator.js +0 -128
- package/dist/testing/conformance/subscriptions/retention.d.ts +0 -4
- package/dist/testing/conformance/subscriptions/retention.js +0 -288
- package/dist/testing/conformance/subscriptions/routing.d.ts +0 -4
- package/dist/testing/conformance/subscriptions/routing.js +0 -179
- package/dist/testing/conformance/subscriptions.d.ts +0 -4
- package/dist/testing/conformance/subscriptions.js +0 -19
- package/dist/testing/conformance/tables.d.ts +0 -125
- package/dist/testing/conformance/tables.js +0 -664
- package/dist/testing/conformance/transports/actors.d.ts +0 -1907
- package/dist/testing/conformance/transports/actors.js +0 -163
- package/dist/testing/conformance/transports/client.d.ts +0 -3
- package/dist/testing/conformance/transports/client.js +0 -237
- package/dist/testing/conformance/transports/feeds.d.ts +0 -3
- package/dist/testing/conformance/transports/feeds.js +0 -232
- package/dist/testing/conformance/transports/harness.d.ts +0 -93
- package/dist/testing/conformance/transports/harness.js +0 -84
- package/dist/testing/conformance/transports/streams.d.ts +0 -3
- package/dist/testing/conformance/transports/streams.js +0 -96
- package/dist/testing/conformance/transports/websocket.d.ts +0 -3
- package/dist/testing/conformance/transports/websocket.js +0 -384
- package/dist/testing/conformance/transports/wire.d.ts +0 -141
- package/dist/testing/conformance/transports/wire.js +0 -204
- package/dist/testing/conformance/transports.d.ts +0 -3
- package/dist/testing/conformance/transports.js +0 -11
- package/dist/testing/conformance/watch.d.ts +0 -59
- package/dist/testing/conformance/watch.js +0 -879
- package/dist/testing/conformance/workflow-engine.d.ts +0 -97
- package/dist/testing/conformance/workflow-engine.js +0 -253
- package/dist/testing/conformance/workflow-versions.d.ts +0 -3
- package/dist/testing/conformance/workflow-versions.js +0 -545
- package/dist/testing/conformance/workflows/access.d.ts +0 -4
- package/dist/testing/conformance/workflows/access.js +0 -147
- package/dist/testing/conformance/workflows/actors.d.ts +0 -2317
- package/dist/testing/conformance/workflows/actors.js +0 -305
- package/dist/testing/conformance/workflows/basics.d.ts +0 -4
- package/dist/testing/conformance/workflows/basics.js +0 -144
- package/dist/testing/conformance/workflows/engine.d.ts +0 -4
- package/dist/testing/conformance/workflows/engine.js +0 -80
- package/dist/testing/conformance/workflows/guards.d.ts +0 -4
- package/dist/testing/conformance/workflows/guards.js +0 -172
- package/dist/testing/conformance/workflows/harness.d.ts +0 -14
- package/dist/testing/conformance/workflows/harness.js +0 -44
- package/dist/testing/conformance/workflows/recovery.d.ts +0 -4
- package/dist/testing/conformance/workflows/recovery.js +0 -181
- package/dist/testing/conformance/workflows/starts.d.ts +0 -4
- package/dist/testing/conformance/workflows/starts.js +0 -57
- package/dist/testing/conformance/workflows/suspension.d.ts +0 -4
- package/dist/testing/conformance/workflows/suspension.js +0 -106
- package/dist/testing/conformance/workflows/waits.d.ts +0 -4
- package/dist/testing/conformance/workflows/waits.js +0 -284
- package/dist/testing/conformance/workflows.d.ts +0 -4
- package/dist/testing/conformance/workflows.js +0 -19
- package/dist/testing/conformance.d.ts +0 -330
- package/dist/testing/conformance.js +0 -364
- package/dist/testing/foundation.d.ts +0 -41
- package/dist/testing/foundation.js +0 -517
- package/dist/testing/simulate-cluster.d.ts +0 -112
- package/dist/testing/simulate-cluster.js +0 -257
- package/dist/testing/simulate.d.ts +0 -90
- package/dist/testing/simulate.js +0 -127
|
@@ -1,257 +0,0 @@
|
|
|
1
|
-
import { Cause, Config, Console, DateTime, Duration, Effect, Exit, Fiber, Option, Predicate, Random, Schedule, Schema, } from "effect";
|
|
2
|
-
import { SqlError } from "effect/sql";
|
|
3
|
-
import { Actor, Actors } from "../index.js";
|
|
4
|
-
import { ActorError } from "../errors/actor.js";
|
|
5
|
-
import { ActorTest } from "./actor-test.js";
|
|
6
|
-
import { ActorCluster } from "./cluster.js";
|
|
7
|
-
import { conclude, generator, SETTLE_ROUNDS } from "./simulate.js";
|
|
8
|
-
const isActorError = Schema.is(ActorError);
|
|
9
|
-
/**
|
|
10
|
-
* Runs `program` on `cluster` under a fault schedule drawn from
|
|
11
|
-
* `options.seed`, then checks that every command committed exactly one
|
|
12
|
-
* receipt and no due or attempted outbox row was left undelivered. A failure
|
|
13
|
-
* dies with the seed. Rerunning it repeats the program's choices and each
|
|
14
|
-
* command's entry runner and fault; which runner owns an actor depends on
|
|
15
|
-
* runner addresses, which change from run to run, and command ids are minted
|
|
16
|
-
* fresh. Settling advances every runner's clock, so simulate once per
|
|
17
|
-
* cluster after any runner has restarted: a runner that starts after the
|
|
18
|
-
* settling has an earlier clock than the others, and a command id minted on a
|
|
19
|
-
* runner ahead of it is in the future to it.
|
|
20
|
-
*/
|
|
21
|
-
export const simulateCluster = (cluster) => (options, program) => Effect.gen(function* () {
|
|
22
|
-
const draw = generator(options.seed);
|
|
23
|
-
const rate = options.faultRate ?? 0.5;
|
|
24
|
-
const attempt = Duration.fromInputUnsafe(options.attempt ?? "10 seconds");
|
|
25
|
-
const within = Duration.fromInputUnsafe(options.within ?? "2 minutes");
|
|
26
|
-
const settle = options.settle ?? "1 minute";
|
|
27
|
-
const steps = [];
|
|
28
|
-
const violations = [];
|
|
29
|
-
const runners = Array.from({ length: cluster.runners }, (_, index) => index);
|
|
30
|
-
const on = cluster.on;
|
|
31
|
-
const planned = options.faults.includes("primaryFailover");
|
|
32
|
-
const others = options.faults.filter((fault) => fault !== "primaryFailover");
|
|
33
|
-
if (runners.length < 2)
|
|
34
|
-
return yield* Effect.die(new Error("ActorTest.simulateCluster needs at least two runners"));
|
|
35
|
-
if (planned && options.primary === undefined)
|
|
36
|
-
return yield* Effect.die(new Error("primaryFailover needs the primary option"));
|
|
37
|
-
if (planned && (options.commands === undefined || options.commands < 1))
|
|
38
|
-
return yield* Effect.die(new Error("primaryFailover needs the number of commands"));
|
|
39
|
-
const failoverAt = planned ? Math.floor(draw() * options.commands) : -1;
|
|
40
|
-
const eachRunner = (effect) => Effect.forEach(runners, (runner) => on(runner)(effect));
|
|
41
|
-
const clearFaults = ActorTest.use((test) => test.clearFaults);
|
|
42
|
-
const clocks = (yield* eachRunner(ActorTest.use((test) => test.now))).map(DateTime.toEpochMillis);
|
|
43
|
-
if (Math.max(...clocks) - Math.min(...clocks) > 1_000)
|
|
44
|
-
return yield* Effect.die(new Error("ActorTest.simulateCluster needs runners whose clocks agree; an earlier simulation settled some runners' clocks ahead of a restarted one, so simulate once per cluster"));
|
|
45
|
-
/** The cluster's own pool reconnects through a failover like any other, so a read that meets one is retried. */
|
|
46
|
-
const read = (query) => query.pipe(Effect.retry({ schedule: Schedule.spaced("200 millis"), times: 50 }), Effect.orDie);
|
|
47
|
-
/**
|
|
48
|
-
* Minting a command id reads the database clock, which a failover
|
|
49
|
-
* briefly takes away, so it is retried. A crash fault is queued on every
|
|
50
|
-
* runner, and one of them takes it; if all of them still hold theirs,
|
|
51
|
-
* the turn never reached its point.
|
|
52
|
-
*/
|
|
53
|
-
const command = (label, effect) => Effect.gen(function* () {
|
|
54
|
-
const fault = steps.length === failoverAt
|
|
55
|
-
? "primaryFailover"
|
|
56
|
-
: others.length === 0 || draw() >= rate
|
|
57
|
-
? "none"
|
|
58
|
-
: others[Math.floor(draw() * others.length)];
|
|
59
|
-
const entry = Math.floor(draw() * runners.length);
|
|
60
|
-
const point = fault === "crashAfterCommit" || (fault === "runnerKill" && draw() < 0.5)
|
|
61
|
-
? "afterCommit"
|
|
62
|
-
: "beforeCommit";
|
|
63
|
-
const landed = (fault === "connectionLoss" || fault === "primaryFailover") && draw() < 0.5;
|
|
64
|
-
const commandId = yield* on(entry)(Effect.gen(function* () {
|
|
65
|
-
return yield* (yield* Actors).mintCommandId;
|
|
66
|
-
})).pipe(Effect.retry({
|
|
67
|
-
while: (error) => error.isRetryable,
|
|
68
|
-
schedule: Schedule.spaced("200 millis"),
|
|
69
|
-
times: 100,
|
|
70
|
-
}), Effect.orDie);
|
|
71
|
-
steps.push({ label, commandId, fault, runner: entry, point, landed });
|
|
72
|
-
const call = effect.pipe(Actor.commandId(commandId));
|
|
73
|
-
const send = (runner) => on(runner)(call).pipe(Effect.timeout(attempt));
|
|
74
|
-
/**
|
|
75
|
-
* Sends the same id through each runner in turn, skipping `avoid`,
|
|
76
|
-
* until a receipt answers; only the command's own declared failures end it.
|
|
77
|
-
*/
|
|
78
|
-
const deliver = (from, avoid) => {
|
|
79
|
-
const order = runners.filter((runner) => runner !== avoid);
|
|
80
|
-
const start = Math.max(order.indexOf(from), 0);
|
|
81
|
-
return Effect.gen(function* () {
|
|
82
|
-
for (let round = 0;; round++) {
|
|
83
|
-
const exit = yield* Effect.exit(send(order[(start + round) % order.length]));
|
|
84
|
-
if (Exit.isSuccess(exit))
|
|
85
|
-
return exit.value;
|
|
86
|
-
const error = Cause.findErrorOption(exit.cause);
|
|
87
|
-
const resendable = Option.isNone(error) ||
|
|
88
|
-
Predicate.isTagged(error.value, "TimeoutError") ||
|
|
89
|
-
(isActorError(error.value) && error.value.isRetryable);
|
|
90
|
-
if (!resendable)
|
|
91
|
-
return yield* Effect.failCause(exit.cause);
|
|
92
|
-
yield* Effect.sleep("100 millis");
|
|
93
|
-
}
|
|
94
|
-
});
|
|
95
|
-
};
|
|
96
|
-
/**
|
|
97
|
-
* Whichever runner runs this command's turn reaches its pause, so
|
|
98
|
-
* every runner holds one and the others' are cleared once the owner is known.
|
|
99
|
-
*/
|
|
100
|
-
const holdTurn = Effect.gen(function* () {
|
|
101
|
-
const paused = yield* Effect.forEach(runners, (runner) => on(runner)(ActorTest.use((test) => test.pauseNext(point, { commandId }))));
|
|
102
|
-
const first = yield* Effect.forkChild(Effect.exit(send(entry)));
|
|
103
|
-
const owner = yield* Effect.raceAll(paused.map((pause, runner) => Effect.as(pause.reached, runner))).pipe(Effect.timeoutOrElse({
|
|
104
|
-
duration: within,
|
|
105
|
-
orElse: () => Effect.die(new Error(`${label} (${commandId}) never reached ${point}`)),
|
|
106
|
-
}));
|
|
107
|
-
yield* Effect.forEach(runners.filter((runner) => runner !== owner), (runner) => on(runner)(clearFaults));
|
|
108
|
-
return { owner, first, release: paused[owner].release };
|
|
109
|
-
});
|
|
110
|
-
const survivor = (owner) => entry === owner ? (owner + 1) % runners.length : entry;
|
|
111
|
-
if (fault === "none")
|
|
112
|
-
return yield* deliver(entry);
|
|
113
|
-
if (fault === "dropReply") {
|
|
114
|
-
yield* Effect.exit(send(entry));
|
|
115
|
-
return yield* deliver(entry);
|
|
116
|
-
}
|
|
117
|
-
if (fault === "crashBeforeCommit" || fault === "crashAfterCommit") {
|
|
118
|
-
yield* eachRunner(ActorTest.use((test) => test.crashNext(point, { commandId })));
|
|
119
|
-
const result = yield* deliver(entry);
|
|
120
|
-
const left = (yield* eachRunner(clearFaults)).flat();
|
|
121
|
-
if (left.length === runners.length)
|
|
122
|
-
violations.push(`${label} (${commandId}) never reached its ${point} crash`);
|
|
123
|
-
return result;
|
|
124
|
-
}
|
|
125
|
-
if (fault === "runnerKill") {
|
|
126
|
-
const { owner, first, release } = yield* holdTurn;
|
|
127
|
-
yield* cluster.kill(owner);
|
|
128
|
-
yield* release;
|
|
129
|
-
const result = yield* deliver(survivor(owner), owner);
|
|
130
|
-
yield* Fiber.interrupt(first);
|
|
131
|
-
yield* cluster.restart(owner);
|
|
132
|
-
yield* cluster.ready;
|
|
133
|
-
return result;
|
|
134
|
-
}
|
|
135
|
-
if (fault === "heartbeatLoss") {
|
|
136
|
-
const { owner, first, release } = yield* holdTurn;
|
|
137
|
-
const heartbeat = yield* cluster.pauseHeartbeat(owner);
|
|
138
|
-
const result = yield* deliver(survivor(owner), owner);
|
|
139
|
-
yield* release;
|
|
140
|
-
yield* Fiber.join(first);
|
|
141
|
-
yield* heartbeat.resume;
|
|
142
|
-
yield* cluster.ready;
|
|
143
|
-
return result;
|
|
144
|
-
}
|
|
145
|
-
const { first, release } = yield* holdTurn;
|
|
146
|
-
if (!landed) {
|
|
147
|
-
yield* cluster.failover;
|
|
148
|
-
yield* release;
|
|
149
|
-
}
|
|
150
|
-
else {
|
|
151
|
-
yield* cluster.holdReplies;
|
|
152
|
-
yield* release;
|
|
153
|
-
yield* read(cluster.sql `
|
|
154
|
-
SELECT count(*)::integer AS count FROM actor_receipts
|
|
155
|
-
WHERE tenant_id = ${cluster.tenant} AND command_id = ${commandId}`).pipe(Effect.repeat({
|
|
156
|
-
until: ([receipts]) => receipts.count > 0,
|
|
157
|
-
schedule: Schedule.spaced("20 millis"),
|
|
158
|
-
}), Effect.timeoutOrElse({
|
|
159
|
-
duration: within,
|
|
160
|
-
orElse: () => Effect.die(new Error(`${label} (${commandId}) never committed`)),
|
|
161
|
-
}));
|
|
162
|
-
yield* cluster.failover;
|
|
163
|
-
}
|
|
164
|
-
if (fault === "primaryFailover")
|
|
165
|
-
yield* options.primary;
|
|
166
|
-
const settled = yield* Fiber.join(first);
|
|
167
|
-
return Exit.isSuccess(settled) ? settled.value : yield* deliver(entry);
|
|
168
|
-
}).pipe(Effect.timeoutOrElse({
|
|
169
|
-
duration: within,
|
|
170
|
-
orElse: () => Effect.die(new Error(`${label} did not commit within ${Duration.format(within)}`)),
|
|
171
|
-
}));
|
|
172
|
-
const simulation = {
|
|
173
|
-
seed: options.seed,
|
|
174
|
-
int: (min, max) => Effect.sync(() => min + Math.floor(draw() * (max - min + 1))),
|
|
175
|
-
pick: (values) => Effect.sync(() => values[Math.floor(draw() * values.length)]),
|
|
176
|
-
command,
|
|
177
|
-
};
|
|
178
|
-
const run = Effect.gen(function* () {
|
|
179
|
-
yield* program(simulation);
|
|
180
|
-
if (steps.length === 0)
|
|
181
|
-
violations.push("the program sent no command");
|
|
182
|
-
if (planned && steps.length <= failoverAt)
|
|
183
|
-
violations.push(`the primary failover planned for command ${failoverAt} never happened`);
|
|
184
|
-
/**
|
|
185
|
-
* A row that was ever attempted is still in flight, while one never
|
|
186
|
-
* attempted and not yet due is a timer the program scheduled.
|
|
187
|
-
*/
|
|
188
|
-
const pending = read(Effect.gen(function* () {
|
|
189
|
-
const now = DateTime.toEpochMillis(yield* on(0)(ActorTest.use((test) => test.now)));
|
|
190
|
-
const [rows] = yield* cluster.sql `
|
|
191
|
-
SELECT count(*)::integer AS count FROM actor_outbox
|
|
192
|
-
WHERE tenant_id = ${cluster.tenant} AND (due_at_ms <= ${now} OR attempts > 0)`;
|
|
193
|
-
return rows.count;
|
|
194
|
-
}));
|
|
195
|
-
let unsettled = Number.POSITIVE_INFINITY;
|
|
196
|
-
for (let round = 0; round < SETTLE_ROUNDS && unsettled > 0; round++) {
|
|
197
|
-
yield* eachRunner(ActorTest.use((test) => test.advance(settle)));
|
|
198
|
-
unsettled = yield* pending.pipe(Effect.repeat({
|
|
199
|
-
until: (count) => count === 0,
|
|
200
|
-
schedule: Schedule.spaced("100 millis"),
|
|
201
|
-
}), Effect.timeoutOrElse({ duration: "5 seconds", orElse: () => pending }));
|
|
202
|
-
}
|
|
203
|
-
if (unsettled > 0)
|
|
204
|
-
violations.push(`${unsettled} due or attempted outbox rows were never delivered`);
|
|
205
|
-
const left = (yield* eachRunner(clearFaults)).flat();
|
|
206
|
-
if (left.length > 0)
|
|
207
|
-
violations.push(`faults at ${left.join(", ")} were never reached`);
|
|
208
|
-
for (const step of steps) {
|
|
209
|
-
const [receipts] = yield* read(cluster.sql `
|
|
210
|
-
SELECT count(*)::integer AS count FROM actor_receipts
|
|
211
|
-
WHERE tenant_id = ${cluster.tenant} AND command_id = ${step.commandId}`);
|
|
212
|
-
if (receipts.count !== 1)
|
|
213
|
-
violations.push(`${step.label} (${step.commandId}) has ${receipts.count} receipts`);
|
|
214
|
-
}
|
|
215
|
-
});
|
|
216
|
-
return yield* conclude({
|
|
217
|
-
seed: options.seed,
|
|
218
|
-
steps,
|
|
219
|
-
violations,
|
|
220
|
-
run,
|
|
221
|
-
cleanup: eachRunner(clearFaults).pipe(Effect.ignore),
|
|
222
|
-
});
|
|
223
|
-
});
|
|
224
|
-
/** Seeds pull requests run on a cluster: `0` and `1`. */
|
|
225
|
-
export const CLUSTER_SIMULATION_SEEDS = 2;
|
|
226
|
-
/** Seeds the nightly run covers on a cluster: about a minute each. */
|
|
227
|
-
export const NIGHTLY_CLUSTER_SIMULATION_SEEDS = 6;
|
|
228
|
-
/** Seeds pull requests run through a real primary failover, each on a primary and standby of its own: `0`. */
|
|
229
|
-
export const FAILOVER_SIMULATION_SEEDS = 1;
|
|
230
|
-
/** Seeds the nightly run puts through a real primary failover: under a minute each. */
|
|
231
|
-
export const NIGHTLY_FAILOVER_SIMULATION_SEEDS = 3;
|
|
232
|
-
const seedsOf = (countName, pullRequest, nightly) => Effect.gen(function* () {
|
|
233
|
-
const property = yield* Config.String("PROPERTY_SEED").pipe(Config.withDefault(""));
|
|
234
|
-
const random = property === "random";
|
|
235
|
-
const count = yield* Config.Int(countName).pipe(Config.withDefault(random ? nightly : pullRequest));
|
|
236
|
-
const configured = yield* Config.Int("SIMULATION_SEED_BASE").pipe(Config.option);
|
|
237
|
-
const base = Option.isSome(configured)
|
|
238
|
-
? configured.value
|
|
239
|
-
: random
|
|
240
|
-
? yield* Random.nextIntBetween(0, 2 ** 31 - 1 - count)
|
|
241
|
-
: 0;
|
|
242
|
-
if (random && Option.isNone(configured))
|
|
243
|
-
yield* Console.error(`${countName} seeds from ${base}`);
|
|
244
|
-
return Array.from({ length: count }, (_, index) => String(base + index));
|
|
245
|
-
}).pipe(Effect.orDie);
|
|
246
|
-
/**
|
|
247
|
-
* The seeds this run simulates on a cluster: `CLUSTER_SIMULATION_SEEDS` of
|
|
248
|
-
* them counted up from `SIMULATION_SEED_BASE`. A run whose `PROPERTY_SEED` is
|
|
249
|
-
* `random`, as the nightly run's is, defaults to
|
|
250
|
-
* `NIGHTLY_CLUSTER_SIMULATION_SEEDS` seeds from a base it draws and prints,
|
|
251
|
-
* so each night covers seeds no other night did; every other run defaults to
|
|
252
|
-
* `CLUSTER_SIMULATION_SEEDS` from 0. A failure names its seed; rerun it with
|
|
253
|
-
* `CLUSTER_SIMULATION_SEEDS=1 SIMULATION_SEED_BASE=<seed>`.
|
|
254
|
-
*/
|
|
255
|
-
export const clusterSimulationSeeds = seedsOf("CLUSTER_SIMULATION_SEEDS", CLUSTER_SIMULATION_SEEDS, NIGHTLY_CLUSTER_SIMULATION_SEEDS);
|
|
256
|
-
/** The seeds for the failover drill, chosen like `clusterSimulationSeeds` from `FAILOVER_SIMULATION_SEEDS`. */
|
|
257
|
-
export const failoverSimulationSeeds = seedsOf("FAILOVER_SIMULATION_SEEDS", FAILOVER_SIMULATION_SEEDS, NIGHTLY_FAILOVER_SIMULATION_SEEDS);
|
|
@@ -1,90 +0,0 @@
|
|
|
1
|
-
import { Duration, Effect } from "effect";
|
|
2
|
-
import { SqlClient } from "effect/sql";
|
|
3
|
-
import { Actors } from "../index.ts";
|
|
4
|
-
import type { ActorTest } from "./actor-test.ts";
|
|
5
|
-
/**
|
|
6
|
-
* A fault `ActorTest.simulate` injects into one command:
|
|
7
|
-
* - `crashBeforeCommit`: the turn dies before COMMIT, so it rolls back and the handle retries.
|
|
8
|
-
* - `crashAfterCommit`: the turn commits and the reply is lost, so the retry reads the receipt.
|
|
9
|
-
* - `dropReply`: the caller discards the reply it got and sends the same command id again.
|
|
10
|
-
* - `relayCrash`: the next relay delivery dies before deleting its outbox row, so the row
|
|
11
|
-
* is redelivered once its claim lease ends. Drawn only for commands marked `relays`.
|
|
12
|
-
* - `clockSkew`: the framework clock jumps ahead of the database clock before the command.
|
|
13
|
-
*/
|
|
14
|
-
export type SimulationFault = "crashBeforeCommit" | "crashAfterCommit" | "dropReply" | "relayCrash" | "clockSkew";
|
|
15
|
-
export interface SimulationOptions {
|
|
16
|
-
/** Every draw, and so the whole fault schedule, follows from it. */
|
|
17
|
-
readonly seed: string;
|
|
18
|
-
readonly faults: ReadonlyArray<SimulationFault>;
|
|
19
|
-
/** Chance that a command gets a fault. Default 0.5. */
|
|
20
|
-
readonly faultRate?: number;
|
|
21
|
-
/** Largest `clockSkew` jump. Default 5 minutes. */
|
|
22
|
-
readonly maxSkew?: Duration.Input;
|
|
23
|
-
/** How far the clock moves to settle outstanding deliveries. Default 1 minute per round. */
|
|
24
|
-
readonly settle?: Duration.Input;
|
|
25
|
-
}
|
|
26
|
-
export interface SimulationStep {
|
|
27
|
-
readonly label: string;
|
|
28
|
-
readonly commandId: string;
|
|
29
|
-
readonly fault: SimulationFault | "none";
|
|
30
|
-
/** The `clockSkew` jump in milliseconds, 0 otherwise. */
|
|
31
|
-
readonly skewMs: number;
|
|
32
|
-
}
|
|
33
|
-
export interface SimulationReport {
|
|
34
|
-
readonly seed: string;
|
|
35
|
-
readonly steps: ReadonlyArray<SimulationStep>;
|
|
36
|
-
}
|
|
37
|
-
export interface Simulation {
|
|
38
|
-
readonly seed: string;
|
|
39
|
-
/**
|
|
40
|
-
* Runs one command under the seed's next fault, with a command id minted
|
|
41
|
-
* for it. `effect` must be a single command call, so it commits one receipt.
|
|
42
|
-
* `relays` marks a command whose turn stages an intent, so a relay crash
|
|
43
|
-
* drawn for it must be reached by that intent's delivery.
|
|
44
|
-
*/
|
|
45
|
-
readonly command: <A, E, R>(label: string, effect: Effect.Effect<A, E, R>, options?: {
|
|
46
|
-
readonly relays?: boolean;
|
|
47
|
-
}) => Effect.Effect<A, E, R>;
|
|
48
|
-
/** A seeded integer in `[min, max]`. */
|
|
49
|
-
readonly int: (min: number, max: number) => Effect.Effect<number>;
|
|
50
|
-
/** A seeded element of a non-empty list. */
|
|
51
|
-
readonly pick: <A>(values: readonly [A, ...Array<A>]) => Effect.Effect<A>;
|
|
52
|
-
}
|
|
53
|
-
/** A small seeded generator, so the schedule never depends on the runtime's own randomness. */
|
|
54
|
-
export declare const generator: (seed: string) => () => number;
|
|
55
|
-
/** Settling rounds before undelivered outbox rows count as lost. */
|
|
56
|
-
export declare const SETTLE_ROUNDS = 5;
|
|
57
|
-
/**
|
|
58
|
-
* Runs a simulation's `run`, then reports it: a defect or a violation dies
|
|
59
|
-
* with the seed and the schedule drawn, and `cleanup` runs however `run` ends.
|
|
60
|
-
*/
|
|
61
|
-
export declare const conclude: <Step extends {
|
|
62
|
-
readonly label: string;
|
|
63
|
-
readonly fault: string;
|
|
64
|
-
}, E, R>({ seed, steps, violations, run, cleanup, }: {
|
|
65
|
-
readonly seed: string;
|
|
66
|
-
readonly steps: ReadonlyArray<Step>;
|
|
67
|
-
readonly violations: ReadonlyArray<string>;
|
|
68
|
-
readonly run: Effect.Effect<void, E, R>;
|
|
69
|
-
readonly cleanup: Effect.Effect<unknown>;
|
|
70
|
-
}) => Effect.Effect<{
|
|
71
|
-
readonly seed: string;
|
|
72
|
-
readonly steps: ReadonlyArray<Step>;
|
|
73
|
-
}, never, R>;
|
|
74
|
-
/**
|
|
75
|
-
* Runs `program` against `test`'s runtime under a fault
|
|
76
|
-
* schedule drawn from `options.seed`, then settles the relay and checks that
|
|
77
|
-
* every command committed exactly one receipt and no due or attempted outbox
|
|
78
|
-
* row was left undelivered. Any failure dies with the seed, so rerunning that
|
|
79
|
-
* seed repeats the program's choices and the fault on each step; command ids
|
|
80
|
-
* are minted fresh on every run. A crashed delivery pushes its row's due time to the end of its claim lease, so an attempted row counts as in flight while an unattempted row that is not yet due is a timer the program scheduled.
|
|
81
|
-
*/
|
|
82
|
-
export declare const simulate: (test: ActorTest["Service"]) => <E, R>(options: SimulationOptions, program: (simulation: Simulation) => Effect.Effect<void, E, R>) => Effect.Effect<SimulationReport, never, R | Actors | SqlClient.SqlClient>;
|
|
83
|
-
/** Seeds pull requests run: `0` to `19`. */
|
|
84
|
-
export declare const SIMULATION_SEEDS = 20;
|
|
85
|
-
/**
|
|
86
|
-
* The seeds this run simulates: `SIMULATION_SEEDS` of them (default 20)
|
|
87
|
-
* counted up from `SIMULATION_SEED_BASE` (default 0), so a nightly run sets
|
|
88
|
-
* both and a failure names the seed to rerun.
|
|
89
|
-
*/
|
|
90
|
-
export declare const simulationSeeds: Effect.Effect<string[], never, never>;
|
package/dist/testing/simulate.js
DELETED
|
@@ -1,127 +0,0 @@
|
|
|
1
|
-
import { Cause, Config, DateTime, Duration, Effect } from "effect";
|
|
2
|
-
import { SqlClient } from "effect/sql";
|
|
3
|
-
import { Actor, Actors } from "../index.js";
|
|
4
|
-
/** A small seeded generator, so the schedule never depends on the runtime's own randomness. */
|
|
5
|
-
export const generator = (seed) => {
|
|
6
|
-
let state = 2166136261;
|
|
7
|
-
for (const char of seed)
|
|
8
|
-
state = Math.imul(state ^ char.charCodeAt(0), 16777619);
|
|
9
|
-
return () => {
|
|
10
|
-
state = (state + 0x6d2b79f5) | 0;
|
|
11
|
-
let mixed = Math.imul(state ^ (state >>> 15), 1 | state);
|
|
12
|
-
mixed = (mixed + Math.imul(mixed ^ (mixed >>> 7), 61 | mixed)) ^ mixed;
|
|
13
|
-
return ((mixed ^ (mixed >>> 14)) >>> 0) / 4294967296;
|
|
14
|
-
};
|
|
15
|
-
};
|
|
16
|
-
const crashPoint = {
|
|
17
|
-
crashBeforeCommit: "beforeCommit",
|
|
18
|
-
crashAfterCommit: "afterCommit",
|
|
19
|
-
relayCrash: "beforeOutboxDelete",
|
|
20
|
-
};
|
|
21
|
-
/** Settling rounds before undelivered outbox rows count as lost. */
|
|
22
|
-
export const SETTLE_ROUNDS = 5;
|
|
23
|
-
const schedule = (steps) => steps.map(({ label, fault }) => `${label}:${fault}`).join(" ");
|
|
24
|
-
/**
|
|
25
|
-
* Runs a simulation's `run`, then reports it: a defect or a violation dies
|
|
26
|
-
* with the seed and the schedule drawn, and `cleanup` runs however `run` ends.
|
|
27
|
-
*/
|
|
28
|
-
export const conclude = ({ seed, steps, violations, run, cleanup, }) => Effect.gen(function* () {
|
|
29
|
-
yield* run.pipe(Effect.ensuring(cleanup), Effect.catchCause((cause) => Effect.die(new Error(`Simulation failed with seed ${seed} after ${steps.length} steps\n${Cause.pretty(cause)}\nschedule: ${schedule(steps)}`))));
|
|
30
|
-
if (violations.length > 0)
|
|
31
|
-
return yield* Effect.die(new Error(`Simulation failed with seed ${seed}\n${violations.join("\n")}\nschedule: ${schedule(steps)}`));
|
|
32
|
-
return { seed, steps };
|
|
33
|
-
});
|
|
34
|
-
/**
|
|
35
|
-
* Runs `program` against `test`'s runtime under a fault
|
|
36
|
-
* schedule drawn from `options.seed`, then settles the relay and checks that
|
|
37
|
-
* every command committed exactly one receipt and no due or attempted outbox
|
|
38
|
-
* row was left undelivered. Any failure dies with the seed, so rerunning that
|
|
39
|
-
* seed repeats the program's choices and the fault on each step; command ids
|
|
40
|
-
* are minted fresh on every run. A crashed delivery pushes its row's due time to the end of its claim lease, so an attempted row counts as in flight while an unattempted row that is not yet due is a timer the program scheduled.
|
|
41
|
-
*/
|
|
42
|
-
export const simulate = (test) => (options, program) => Effect.gen(function* () {
|
|
43
|
-
const actors = yield* Actors;
|
|
44
|
-
const sql = yield* SqlClient.SqlClient;
|
|
45
|
-
const draw = generator(options.seed);
|
|
46
|
-
const rate = options.faultRate ?? 0.5;
|
|
47
|
-
const maxSkewMs = Duration.toMillis(Duration.fromInputUnsafe(options.maxSkew ?? "5 minutes"));
|
|
48
|
-
const settle = options.settle ?? "1 minute";
|
|
49
|
-
const steps = [];
|
|
50
|
-
const int = (min, max) => Effect.sync(() => min + Math.floor(draw() * (max - min + 1)));
|
|
51
|
-
const direct = options.faults.filter((fault) => fault !== "relayCrash");
|
|
52
|
-
const nextFault = (relays) => Effect.sync(() => {
|
|
53
|
-
const faults = relays ? options.faults : direct;
|
|
54
|
-
if (faults.length === 0 || draw() >= rate)
|
|
55
|
-
return "none";
|
|
56
|
-
return faults[Math.floor(draw() * faults.length)];
|
|
57
|
-
});
|
|
58
|
-
const simulation = {
|
|
59
|
-
seed: options.seed,
|
|
60
|
-
int,
|
|
61
|
-
pick: (values) => Effect.sync(() => values[Math.floor(draw() * values.length)]),
|
|
62
|
-
command: (label, effect, commandOptions) => Effect.gen(function* () {
|
|
63
|
-
const fault = yield* nextFault(commandOptions?.relays === true);
|
|
64
|
-
const skewMs = fault === "clockSkew" ? Math.floor(draw() * maxSkewMs) : 0;
|
|
65
|
-
const commandId = yield* actors.mintCommandId.pipe(Effect.orDie);
|
|
66
|
-
steps.push({ label, commandId, fault, skewMs });
|
|
67
|
-
const call = effect.pipe(Actor.commandId(commandId));
|
|
68
|
-
const point = crashPoint[fault];
|
|
69
|
-
if (point !== undefined)
|
|
70
|
-
yield* test.crashNext(point);
|
|
71
|
-
if (skewMs > 0)
|
|
72
|
-
yield* test.advance(skewMs);
|
|
73
|
-
if (fault === "dropReply")
|
|
74
|
-
yield* Effect.exit(call);
|
|
75
|
-
return yield* call;
|
|
76
|
-
}),
|
|
77
|
-
};
|
|
78
|
-
const violations = [];
|
|
79
|
-
const run = Effect.gen(function* () {
|
|
80
|
-
yield* program(simulation);
|
|
81
|
-
if (steps.length === 0)
|
|
82
|
-
violations.push("the program sent no command");
|
|
83
|
-
let unsettled = Number.POSITIVE_INFINITY;
|
|
84
|
-
for (let round = 0; round < SETTLE_ROUNDS && unsettled > 0; round++) {
|
|
85
|
-
yield* test.advance(settle);
|
|
86
|
-
const now = DateTime.toEpochMillis(yield* test.now);
|
|
87
|
-
const [pending] = yield* sql `
|
|
88
|
-
SELECT count(*)::integer AS count FROM actor_outbox
|
|
89
|
-
WHERE tenant_id = ${test.tenant} AND (due_at_ms <= ${now} OR attempts > 0)`;
|
|
90
|
-
unsettled = pending.count;
|
|
91
|
-
}
|
|
92
|
-
if (unsettled > 0)
|
|
93
|
-
violations.push(`${unsettled} due or attempted outbox rows were never delivered`);
|
|
94
|
-
const left = yield* test.clearFaults;
|
|
95
|
-
if (left.length > 0)
|
|
96
|
-
violations.push(`faults at ${left.join(", ")} were never reached`);
|
|
97
|
-
for (const step of steps) {
|
|
98
|
-
const [receipts] = yield* sql `
|
|
99
|
-
SELECT count(*)::integer AS count FROM actor_receipts
|
|
100
|
-
WHERE tenant_id = ${test.tenant} AND command_id = ${step.commandId}`;
|
|
101
|
-
if (receipts.count !== 1)
|
|
102
|
-
violations.push(`${step.label} (${step.commandId}) has ${receipts.count} receipts`);
|
|
103
|
-
}
|
|
104
|
-
});
|
|
105
|
-
const queued = yield* test.clearFaults;
|
|
106
|
-
if (queued.length > 0)
|
|
107
|
-
return yield* Effect.die(new Error(`ActorTest.simulate injects every fault itself; faults at ${queued.join(", ")} were already queued`));
|
|
108
|
-
return yield* conclude({
|
|
109
|
-
seed: options.seed,
|
|
110
|
-
steps,
|
|
111
|
-
violations,
|
|
112
|
-
run,
|
|
113
|
-
cleanup: test.clearFaults,
|
|
114
|
-
});
|
|
115
|
-
});
|
|
116
|
-
/** Seeds pull requests run: `0` to `19`. */
|
|
117
|
-
export const SIMULATION_SEEDS = 20;
|
|
118
|
-
/**
|
|
119
|
-
* The seeds this run simulates: `SIMULATION_SEEDS` of them (default 20)
|
|
120
|
-
* counted up from `SIMULATION_SEED_BASE` (default 0), so a nightly run sets
|
|
121
|
-
* both and a failure names the seed to rerun.
|
|
122
|
-
*/
|
|
123
|
-
export const simulationSeeds = Effect.gen(function* () {
|
|
124
|
-
const count = yield* Config.Int("SIMULATION_SEEDS").pipe(Config.withDefault(SIMULATION_SEEDS));
|
|
125
|
-
const base = yield* Config.Int("SIMULATION_SEED_BASE").pipe(Config.withDefault(0));
|
|
126
|
-
return Array.from({ length: count }, (_, index) => String(base + index));
|
|
127
|
-
}).pipe(Effect.orDie);
|