tigerbeetle 0.0.40 → 0.17.8

data/ext/tb_client/tigerbeetle/src/testing/cluster.zig

@@ -1,1198 +0,0 @@
-const std = @import("std");
-const assert = std.debug.assert;
-const maybe = stdx.maybe;
-const mem = std.mem;
-const log = std.log.scoped(.cluster);
-
-const stdx = @import("stdx");
-const Ratio = stdx.PRNG.Ratio;
-
-const constants = @import("../constants.zig");
-const message_pool = @import("../message_pool.zig");
-const ratio = stdx.PRNG.ratio;
-const MessagePool = message_pool.MessagePool;
-const Message = MessagePool.Message;
-const IO = @import("io.zig").IO;
-
-const AOF = @import("../aof.zig").AOFType(IO);
-const TimeSim = @import("time.zig").TimeSim;
-const Multiversion = vsr.multiversion.Multiversion;
-const IdPermutation = @import("id.zig").IdPermutation;
-
-const StateCheckerType = @import("cluster/state_checker.zig").StateCheckerType;
-const StorageChecker = @import("cluster/storage_checker.zig").StorageChecker;
-const GridChecker = @import("cluster/grid_checker.zig").GridChecker;
-const ManifestCheckerType = @import("cluster/manifest_checker.zig").ManifestCheckerType;
-const JournalCheckerType = @import("cluster/journal_checker.zig").JournalCheckerType;
-
-const vsr = @import("../vsr.zig");
-const format_writes_max = @import("../vsr/replica_format.zig").writes_max;
-
-const MiB = stdx.MiB;
-
-pub const ReplicaHealth = union(enum) {
-    up: struct { paused: bool },
-    down,
-    reformatting,
-};
-
-pub const Release = struct {
-    release: vsr.Release,
-    release_client_min: vsr.Release,
-};
-
-/// Integer values represent exit codes.
-// TODO This doesn't really belong in Cluster, but it is needed here so that StateChecker failures
-// use the particular exit code.
-pub const Failure = enum(u8) {
-    /// Any assertion crash will be given an exit code of 127 by default.
-    crash = 127,
-    liveness = 128,
-    correctness = 129,
-};
-
-/// Shift the id-generating index because the simulator network expects client ids to never collide
-/// with a replica index.
-const client_id_permutation_shift = constants.members_max;
-
-pub fn ClusterType(comptime StateMachineType: anytype) type {
-    return struct {
-        const Cluster = @This();
-
-        pub const Network = @import("cluster/network.zig").Network;
-        pub const NetworkOptions = @import("cluster/network.zig").NetworkOptions;
-        pub const Storage = @import("storage.zig").Storage;
-        pub const StorageFaultAtlas = @import("storage.zig").ClusterFaultAtlas;
-        pub const Tracer = Storage.Tracer;
-        pub const SuperBlock = vsr.SuperBlockType(Storage);
-        pub const MessageBus = @import("cluster/message_bus.zig").MessageBus;
-        pub const StateMachine = StateMachineType(Storage);
-        pub const Replica = vsr.ReplicaType(StateMachine, MessageBus, Storage, AOF);
-        pub const ReplicaReformat =
-            vsr.ReplicaReformatType(StateMachine, MessageBus, Storage);
-        pub const Client = vsr.ClientType(StateMachine.Operation, MessageBus);
-        pub const StateChecker = StateCheckerType(Client, Replica);
-        pub const ManifestChecker = ManifestCheckerType(StateMachine.Forest);
-        pub const JournalChecker = JournalCheckerType(Replica);
-
-        pub const Options = struct {
-            cluster_id: u128,
-            replica_count: u8,
-            standby_count: u8,
-            client_count: u8,
-            storage_size_limit: u64,
-            reformats_max: u8,
-            seed: u64,
-            /// A monotonically-increasing list of releases.
-            /// Initially:
-            /// - All replicas are formatted and started with releases[0].
-            /// - Only releases[0] is "bundled" in each replica. (Use `replica_restart()` to add
-            ///   more).
-            releases: []const Release,
-            client_release: vsr.Release,
-            state_machine: StateMachine.Options,
-            replicate_options: Replica.ReplicateOptions = .{},
-        };
-
-        pub const Callbacks = struct {
-            /// Invoked when a replica produces a reply.
-            /// Includes operation=register messages.
-            /// `client` is null when the prepare does not originate from a client.
-            on_cluster_reply: ?*const fn (
-                cluster: *Cluster,
-                client: ?usize,
-                prepare: *const Message.Prepare,
-                reply: *const Message.Reply,
-            ) void = null,
-
-            /// Invoked when a client receives a reply.
-            /// Includes operation=register messages.
-            on_client_reply: ?*const fn (
-                cluster: *Cluster,
-                client: usize,
-                request: *const Message.Request,
-                reply: *const Message.Reply,
-            ) void = null,
-        };
-
-        allocator: mem.Allocator,
-        prng: stdx.PRNG,
-        options: Options,
-        callbacks: Callbacks,
-
-        network: *Network,
-        storages: []Storage,
-        storage_fault_atlas: *StorageFaultAtlas,
-
-        aofs: []AOF,
-        aof_ios: []IO,
-        aof_io_files: [][1]IO.File,
-
-        /// NB: includes both active replicas and standbys.
-        replicas: []Replica,
-        replica_pools: []MessagePool,
-        replica_times: []TimeSim,
-        replica_tracers: []Tracer,
-        replica_health: []ReplicaHealth,
-        replica_upgrades: []?vsr.Release,
-        replica_reformats: []?ReplicaReformat,
-        replica_releases_bundled: []vsr.ReleaseList,
-        replica_pipeline_requests_limit: u32,
-        replica_count: u8,
-        standby_count: u8,
-        reformat_count: u32 = 0,
-
-        clients: []?Client,
-        client_pools: []MessagePool,
-        client_times: []TimeSim,
-        /// Updated when the *client* is informed of the eviction.
-        /// (Which may be some time after the client is actually evicted by the cluster.)
-        client_eviction_reasons: []?vsr.Header.Eviction.Reason,
-        client_eviction_requests_cancelled: u32 = 0,
-
-        client_id_permutation: IdPermutation,
-
-        state_checker: StateChecker,
-        storage_checker: StorageChecker,
-        grid_checker: *GridChecker,
-        manifest_checker: ManifestChecker,
-
-        context: ?*anyopaque = null,
-
-        pub fn init(
-            allocator: mem.Allocator,
-            options: struct {
-                cluster: Options,
-                network: NetworkOptions,
-                storage: Storage.Options,
-                storage_fault_atlas: StorageFaultAtlas.Options,
-                callbacks: Callbacks,
-            },
-        ) !*Cluster {
-            assert(options.cluster.replica_count >= 1);
-            assert(options.cluster.replica_count <= 6);
-            assert(options.cluster.client_count > 0);
-            assert(options.cluster.storage_size_limit % constants.sector_size == 0);
-            assert(options.cluster.storage_size_limit <= constants.storage_size_limit_max);
-            assert(options.cluster.releases.len > 0);
-            assert(options.storage.replica_index == null);
-            assert(options.storage.fault_atlas == null);
-
-            for (
-                options.cluster.releases[0 .. options.cluster.releases.len - 1],
-                options.cluster.releases[1..],
-            ) |release_a, release_b| {
-                assert(release_a.release.value < release_b.release.value);
-                assert(release_a.release_client_min.value <= release_b.release.value);
-                assert(release_a.release_client_min.value <= release_b.release_client_min.value);
-            }
-
-            const client_count_total = options.cluster.client_count + options.cluster.reformats_max;
-            const node_count = options.cluster.replica_count + options.cluster.standby_count;
-
-            var prng = stdx.PRNG.from_seed(options.cluster.seed);
-
-            // TODO(Zig) Client.init()'s MessagePool.Options require a reference to the network.
-            // Use @returnAddress() instead.
-            var network = try allocator.create(Network);
-            errdefer allocator.destroy(network);
-
-            var network_options = options.network;
-            network_options.client_count += options.cluster.reformats_max;
-            network.* = try Network.init(allocator, network_options);
-            errdefer network.deinit();
-
-            const storage_fault_atlas = try allocator.create(StorageFaultAtlas);
-            errdefer allocator.destroy(storage_fault_atlas);
-
-            storage_fault_atlas.* = try StorageFaultAtlas.init(
-                allocator,
-                options.cluster.replica_count,
-                &prng,
-                options.storage_fault_atlas,
-            );
-            errdefer storage_fault_atlas.deinit(allocator);
-
-            var grid_checker = try allocator.create(GridChecker);
-            errdefer allocator.destroy(grid_checker);
-
-            grid_checker.* = GridChecker.init(allocator);
-            errdefer grid_checker.deinit();
-
-            const storages = try allocator.alloc(Storage, node_count);
-            errdefer allocator.free(storages);
-
-            for (storages, 0..) |*storage, replica_index| {
-                errdefer for (storages[0..replica_index]) |*s| s.deinit(allocator);
-                var storage_options = options.storage;
-                storage_options.replica_index = @intCast(replica_index);
-                storage_options.fault_atlas = storage_fault_atlas;
-                storage_options.grid_checker = grid_checker;
-                storage_options.iops_write_max = @max(format_writes_max, constants.iops_write_max);
-                storage.* = try Storage.init(allocator, storage_options);
-                // Disable most faults at startup,
-                // so that the replicas don't get stuck recovering_head.
-                storage.faulty =
-                    replica_index >= vsr.quorums(options.cluster.replica_count).view_change;
-            }
-            errdefer for (storages) |*storage| storage.deinit(allocator);
-
-            var replica_pools = try allocator.alloc(MessagePool, node_count);
-            errdefer allocator.free(replica_pools);
-
-            // There may be more clients than `clients_max` (to test session eviction).
-            // +1 is for pulse which uses client_id = 0.
-            const pipeline_requests_limit =
-                (@min(options.cluster.client_count, constants.clients_max) + @as(u8, 1)) -|
-                constants.pipeline_prepare_queue_max;
-
-            for (replica_pools, 0..) |*pool, i| {
-                errdefer for (replica_pools[0..i]) |*p| p.deinit(allocator);
-                pool.* = try MessagePool.init(allocator, .{ .replica = .{
-                    .members_count = options.cluster.replica_count + options.cluster.standby_count,
-                    .pipeline_requests_limit = pipeline_requests_limit,
-                    .message_bus = .testing,
-                } });
-            }
-            errdefer for (replica_pools) |*pool| pool.deinit(allocator);
-
-            const replica_times = try allocator.alloc(TimeSim, node_count);
-            errdefer allocator.free(replica_times);
-            @memset(replica_times, .{
-                .resolution = constants.tick_ms * std.time.ns_per_ms,
-                .offset_type = .linear,
-                .offset_coefficient_A = 0,
-                .offset_coefficient_B = 0,
-            });
-
-            const replica_tracers = try allocator.alloc(Tracer, node_count);
-            errdefer allocator.free(replica_tracers);
-
-            for (replica_tracers, 0..) |*tracer, replica_index| {
-                errdefer for (replica_tracers[0..replica_index]) |*t| t.deinit(allocator);
-                const time = replica_times[replica_index].time();
-                tracer.* = try Tracer.init(allocator, time, .{ .replica = .{
-                    .cluster = options.cluster.cluster_id,
-                    .replica = @intCast(replica_index),
-                } }, .{});
-            }
-            errdefer for (replica_tracers) |*tracer| tracer.deinit(allocator);
-
-            const replicas = try allocator.alloc(Replica, node_count);
-            errdefer allocator.free(replicas);
-
-            const replica_health = try allocator.alloc(ReplicaHealth, node_count);
-            errdefer allocator.free(replica_health);
-            @memset(replica_health, .{ .up = .{ .paused = false } });
-
-            const replica_upgrades = try allocator.alloc(?vsr.Release, node_count);
-            errdefer allocator.free(replica_upgrades);
-            @memset(replica_upgrades, null);
-
-            const replica_reformats =
-                try allocator.alloc(?ReplicaReformat, options.cluster.replica_count);
-            errdefer allocator.free(replica_reformats);
-            @memset(replica_reformats, null);
-
-            var client_pools = try allocator.alloc(MessagePool, client_count_total);
-            errdefer allocator.free(client_pools);
-
-            for (client_pools, 0..) |*pool, i| {
-                errdefer for (client_pools[0..i]) |*p| p.deinit(allocator);
-                pool.* = try MessagePool.init(allocator, .client);
-            }
-            errdefer for (client_pools) |*pool| pool.deinit(allocator);
-
-            const client_eviction_reasons =
-                try allocator.alloc(?vsr.Header.Eviction.Reason, client_count_total);
-            errdefer allocator.free(client_eviction_reasons);
-            @memset(client_eviction_reasons, null);
-
-            const client_times = try allocator.alloc(TimeSim, client_count_total);
-            errdefer allocator.free(client_times);
-            @memset(client_times, .{
-                .resolution = constants.tick_ms * std.time.ns_per_ms,
-                .offset_type = .linear,
-                .offset_coefficient_A = 0,
-                .offset_coefficient_B = 0,
-            });
-
-            const client_id_permutation = IdPermutation.generate(&prng);
-            var clients = try allocator.alloc(?Client, client_count_total);
-            errdefer allocator.free(clients);
-
-            for (clients, 0..) |*client, i| {
-                errdefer for (clients[0..i]) |*c| c.*.?.deinit(allocator);
-                client.* = try Client.init(
-                    allocator,
-                    client_times[i].time(),
-                    &client_pools[i],
-                    .{
-                        .id = client_id_permutation.encode(i + client_id_permutation_shift),
-                        .cluster = options.cluster.cluster_id,
-                        .replica_count = options.cluster.replica_count,
-                        .aof_recovery = false,
-                        .message_bus_options = .{ .network = network },
-                        .eviction_callback = client_on_eviction,
-                    },
-                );
-                client.*.?.release = options.cluster.client_release;
-            }
-            errdefer for (clients) |*client| client.*.?.deinit(allocator);
-
-            var state_checker = try StateChecker.init(allocator, .{
-                .cluster_id = options.cluster.cluster_id,
-                .replicas = replicas,
-                .replica_count = options.cluster.replica_count,
-                .clients = clients,
-            });
-            errdefer state_checker.deinit();
-
-            var storage_checker = try StorageChecker.init(allocator);
-            errdefer storage_checker.deinit(allocator);
-
-            var manifest_checker = ManifestChecker.init(allocator);
-            errdefer manifest_checker.deinit();
-
-            // Format each replica's storage (equivalent to "tigerbeetle format ...").
-            for (storages, 0..) |*storage, replica_index| {
-                try vsr.format(
-                    Storage,
-                    allocator,
-                    storage,
-                    .{
-                        .cluster = options.cluster.cluster_id,
-                        .release = options.cluster.releases[0].release,
-                        .replica = @intCast(replica_index),
-                        .replica_count = options.cluster.replica_count,
-                        .view = null,
-                    },
-                );
-            }
-
-            const replica_releases_bundled = try allocator.alloc(vsr.ReleaseList, node_count);
-            errdefer allocator.free(replica_releases_bundled);
-
-            // We must heap-allocate the cluster since its pointer will be attached to the replica.
-            // TODO(Zig) @returnAddress().
-            var cluster = try allocator.create(Cluster);
-            errdefer allocator.destroy(cluster);
-
-            cluster.aofs = try allocator.alloc(AOF, node_count);
-            errdefer allocator.free(cluster.aofs);
-
-            cluster.aof_io_files = try allocator.alloc([1]IO.File, node_count);
-            errdefer allocator.free(cluster.aof_io_files);
-
-            cluster.aof_ios = try allocator.alloc(IO, node_count);
-            errdefer allocator.free(cluster.aof_ios);
-
-            for (
-                cluster.aofs,
-                cluster.aof_ios,
-                cluster.aof_io_files,
-                0..,
-            ) |*aof, *aof_io, *aof_io_file, i| {
-                const buffer = try allocator.alignedAlloc(
-                    u8,
-                    constants.sector_size,
-                    // Arbitrary value.
-                    32 * MiB,
-                );
-                errdefer allocator.free(buffer);
-
-                aof_io_file[0] = .{ .buffer = buffer };
-                aof_io.* = try IO.init(aof_io_file, .{
-                    .seed = options.cluster.seed,
-                    .larger_than_logical_sector_read_fault_probability = Ratio.zero(),
-                });
-                errdefer for (cluster.aof_ios[0..i]) |*io| io.deinit();
-
-                aof.* = AOF{
-                    .io = aof_io,
-                    .path = "test.aof",
-                    .fd = 0,
-                };
-                errdefer for (cluster.aofs[0..i]) |*aof_| aof_.deinit(allocator);
-            }
-
-            cluster.* = Cluster{
-                .allocator = allocator,
-                .prng = prng,
-                .options = options.cluster,
-                .callbacks = options.callbacks,
-                .network = network,
-                .storages = storages,
-                .aofs = cluster.aofs,
-                .aof_ios = cluster.aof_ios,
-                .aof_io_files = cluster.aof_io_files,
-                .storage_fault_atlas = storage_fault_atlas,
-                .replicas = replicas,
-                .replica_pools = replica_pools,
-                .replica_times = replica_times,
-                .replica_tracers = replica_tracers,
-                .replica_health = replica_health,
-                .replica_upgrades = replica_upgrades,
-                .replica_reformats = replica_reformats,
-                .replica_pipeline_requests_limit = pipeline_requests_limit,
-                .replica_releases_bundled = replica_releases_bundled,
-                .replica_count = options.cluster.replica_count,
-                .standby_count = options.cluster.standby_count,
-                .clients = clients,
-                .client_pools = client_pools,
-                .client_times = client_times,
-                .client_eviction_reasons = client_eviction_reasons,
-                .client_id_permutation = client_id_permutation,
-                .state_checker = state_checker,
-                .storage_checker = storage_checker,
-                .grid_checker = grid_checker,
-                .manifest_checker = manifest_checker,
-            };
-
-            for (cluster.replicas, 0..) |_, replica_index| {
-                errdefer for (replicas[0..replica_index]) |*r| r.deinit(allocator);
-
-                cluster.replica_releases_bundled[replica_index] = .empty;
-                cluster.replica_releases_bundled[replica_index].push(
-                    options.cluster.releases[0].release,
-                );
-
-                // Nonces are incremented on restart, so spread them out across 128 bit space
-                // to avoid collisions.
-                const nonce = (@as(u128, replica_index) << 64) + 1;
-                try cluster.replica_open(@intCast(replica_index), .{
-                    .nonce = nonce,
-                    .release = options.cluster.releases[0].release,
-                });
-            }
-            errdefer for (cluster.replicas) |*replica| replica.deinit(allocator);
-
-            for (clients) |*client| {
-                client.*.?.on_reply_context = cluster;
-                client.*.?.on_reply_callback = client_on_reply;
-                network.link(client.*.?.message_bus.process, &client.*.?.message_bus);
-            }
-
-            return cluster;
-        }
-
-        pub fn deinit(cluster: *Cluster) void {
-            cluster.manifest_checker.deinit();
-            cluster.storage_checker.deinit(cluster.allocator);
-            cluster.state_checker.deinit();
-            cluster.network.deinit();
-
-            for (cluster.clients) |*client_maybe| {
-                if (client_maybe.*) |*client| {
-                    client.deinit(cluster.allocator);
-                }
-            }
-
-            for (cluster.client_pools) |*pool| pool.deinit(cluster.allocator);
-            for (cluster.replicas, 0..) |*replica, i| {
-                switch (cluster.replica_health[i]) {
-                    .up => replica.deinit(cluster.allocator),
-                    .down => {},
-                    .reformatting => cluster.replica_reformats[i].?.deinit(cluster.allocator),
-                }
-            }
-            for (cluster.replica_tracers) |*tracer| tracer.deinit(cluster.allocator);
-            for (cluster.replica_pools) |*pool| pool.deinit(cluster.allocator);
-            for (cluster.storages) |*storage| storage.deinit(cluster.allocator);
-
-            for (cluster.aofs) |*aof| aof.close();
-
-            for (cluster.aof_ios) |*io| io.deinit();
-            cluster.allocator.free(cluster.aof_ios);
-
-            for (cluster.aof_io_files) |*io_file| {
-                for (io_file) |file| cluster.allocator.free(file.buffer);
-            }
-            cluster.allocator.free(cluster.aof_io_files);
-
-            cluster.storage_fault_atlas.deinit(cluster.allocator);
-            cluster.grid_checker.deinit(); // (Storage references this.)
-
-            cluster.allocator.free(cluster.clients);
-            cluster.allocator.free(cluster.client_times);
-            cluster.allocator.free(cluster.client_eviction_reasons);
-            cluster.allocator.free(cluster.client_pools);
-            cluster.allocator.free(cluster.replicas);
-            cluster.allocator.free(cluster.replica_reformats);
-            cluster.allocator.free(cluster.replica_upgrades);
-            cluster.allocator.free(cluster.replica_health);
-            cluster.allocator.free(cluster.replica_times);
-            cluster.allocator.free(cluster.replica_tracers);
-            cluster.allocator.free(cluster.replica_pools);
-            cluster.allocator.free(cluster.storages);
-            cluster.allocator.free(cluster.aofs);
-            cluster.allocator.free(cluster.replica_releases_bundled);
-            cluster.allocator.destroy(cluster.grid_checker);
-            cluster.allocator.destroy(cluster.storage_fault_atlas);
-            cluster.allocator.destroy(cluster.network);
-            cluster.allocator.destroy(cluster);
-        }
-
-        pub fn tick(cluster: *Cluster) void {
-            // Interleave storage and network steps, to allow for faster-than-a-tick IO.
-            while (true) {
-                var advanced = false;
-                advanced = cluster.network.step() or advanced;
-
-                for (cluster.clients, cluster.client_eviction_reasons) |*client, eviction_reason| {
-                    if (client.* != null and eviction_reason != null) {
-                        client.*.?.deinit(cluster.allocator);
-                        client.* = null;
-                    }
-                }
-
-                for (
-                    cluster.storages,
-                    cluster.replica_health,
-                    cluster.replica_upgrades,
-                    0..,
-                ) |*storage, *health, *upgrade, i| {
-                    if (health.* == .up and health.*.up.paused) continue;
-                    // Upgrades immediately follow storage.step(), since upgrades occur at
-                    // checkpoint completion. (Downgrades are triggered separately – see
-                    // replica_restart()).
-                    advanced = storage.step() or advanced;
-                    if (upgrade.*) |_| cluster.replica_release_execute(@intCast(i));
-                    assert(upgrade.* == null);
-                }
-
-                if (!advanced) break;
-            }
-
-            cluster.network.tick();
-
-            for (cluster.clients) |*client_maybe| {
-                if (client_maybe.*) |*client| client.tick();
-            }
-
-            for (
-                cluster.storages,
-                cluster.replicas,
-                cluster.aof_ios,
-                cluster.replica_times,
-                cluster.replica_health,
-                0..,
-            ) |*storage, *replica, *aof_io, *time_sim, *health, replica_index| {
-                const time = time_sim.time();
-
-                if (health.* == .up and health.*.up.paused) {
-                    // Tick the time even in a paused state, to simulate VM migration.
-                    time.tick();
-                } else {
-                    storage.tick();
-                    switch (health.*) {
-                        .reformatting => {
-                            cluster.tick_reformat(@intCast(replica_index));
-                            time.tick();
-                        },
-                        .up => |up| {
-                            assert(!up.paused);
-
-                            replica.tick();
-                            aof_io.run() catch |err| {
-                                std.debug.panic("{}: io.run() failed: error={}", .{
-                                    replica.replica,
-                                    err,
-                                });
-                            };
-
-                            // For performance, don't run every tick.
-                            if (cluster.prng.chance(ratio(1, 100))) {
-                                JournalChecker.check(replica);
-                            }
-
-                            cluster.state_checker.check_state(replica.replica) catch |err| {
-                                fatal(.correctness, "state checker error: {}", .{err});
-                            };
-                        },
-                        .down => {
-                            // Keep ticking the time so that it won't have diverged too far to
-                            // synchronize when the replica restarts.
-                            time.tick();
-                        },
-                    }
-                }
-            }
-        }
-
-        fn tick_reformat(cluster: *Cluster, replica_index: u8) void {
-            assert(cluster.replica_health[replica_index] == .reformatting);
-
-            const reformat = &cluster.replica_reformats[replica_index].?;
-            const result = reformat.done() orelse return;
-            assert(result == .ok);
-
-            reformat.deinit(cluster.allocator);
-            cluster.replica_reformats[replica_index] = null;
-            cluster.replica_health[replica_index] = .down;
-            cluster.replica_restart(replica_index) catch unreachable;
-            cluster.state_checker.reformat(replica_index);
-        }
-
-        pub fn replica_set_releases(
-            cluster: *Cluster,
-            replica_index: u8,
-            releases: *const vsr.ReleaseList,
-        ) void {
-            cluster.replica_releases_bundled[replica_index] = releases.*;
-        }
-
-        pub fn replica_pause(cluster: *Cluster, replica_index: u8) void {
-            assert(cluster.replica_health[replica_index] == .up);
-            assert(!cluster.replica_health[replica_index].up.paused);
-            cluster.replica_health[replica_index].up.paused = true;
-        }
-
-        pub fn replica_unpause(cluster: *Cluster, replica_index: u8) void {
-            assert(cluster.replica_health[replica_index] == .up);
-            assert(cluster.replica_health[replica_index].up.paused);
-            cluster.replica_health[replica_index].up.paused = false;
-        }
-
-        /// Returns an error when the replica was unable to recover (open).
-        pub fn replica_restart(
-            cluster: *Cluster,
-            replica_index: u8,
-        ) !void {
-            assert(cluster.replica_health[replica_index] == .down);
-            assert(cluster.replica_upgrades[replica_index] == null);
-
-            defer maybe(cluster.replica_health[replica_index] == .up);
-            defer assert(cluster.replica_upgrades[replica_index] == null);
-
-            try cluster.replica_open(replica_index, .{
-                .nonce = cluster.replicas[replica_index].nonce + 1,
-                .release = cluster.replica_releases_bundled[replica_index].last(),
-            });
-            cluster.replica_enable(replica_index);
-
-            if (cluster.replica_upgrades[replica_index]) |_| {
-                // Upgrade the replica promptly, rather than waiting until the next tick().
-                // This ensures that the restart completes synchronously, as the caller expects.
-                cluster.replica_release_execute(replica_index);
-            }
-        }
-
-        /// Reset a replica to its initial state, simulating a random crash/panic.
-        /// Leave the persistent storage untouched, and leave any currently
-        /// inflight messages to/from the replica in the network.
-        pub fn replica_crash(cluster: *Cluster, replica_index: u8) void {
-            assert(cluster.replica_health[replica_index] == .up);
-
-            // Reset the storage before the replica so that pending writes can (partially) finish.
-            cluster.storages[replica_index].reset();
-
-            cluster.replicas[replica_index].deinit(cluster.allocator);
-            cluster.network.process_disable(.{ .replica = replica_index });
-            cluster.replica_health[replica_index] = .down;
-            cluster.log_replica(.crash, replica_index);
-
-            // Ensure that none of the replica's messages leaked when it was deinitialized.
-            const message_bus = cluster.network.get_message_bus(.{ .replica = replica_index });
-            assert(message_bus.pool.free_list.count() == message_bus.pool.messages_max);
-        }
-
-        fn replica_enable(cluster: *Cluster, replica_index: u8) void {
-            assert(cluster.replica_health[replica_index] == .down);
-
-            cluster.network.process_enable(.{ .replica = replica_index });
-            cluster.replica_health[replica_index] = .{ .up = .{ .paused = false } };
-            cluster.log_replica(.recover, replica_index);
-        }
-
-        fn replica_open(cluster: *Cluster, replica_index: u8, options: struct {
-            nonce: u128,
-            release: vsr.Release,
-        }) !void {
-            const release_client_min = for (cluster.options.releases) |release| {
-                if (release.release.value == options.release.value) {
-                    break release.release_client_min;
-                }
-            } else unreachable;
-
-            // Re-initialize the trace to get a clean state.
-            cluster.replica_tracers[replica_index].deinit(cluster.allocator);
-            cluster.replica_tracers[replica_index] = try Tracer.init(
-                cluster.allocator,
-                cluster.replica_times[replica_index].time(),
-                .{ .replica = .{
-                    .cluster = cluster.replicas[replica_index].cluster,
-                    .replica = @intCast(replica_index),
-                } },
-                .{},
-            );
-
-            cluster.aofs[replica_index].reset();
-            cluster.aof_ios[replica_index].reset();
-            var replica = &cluster.replicas[replica_index];
-            try replica.open(
-                cluster.allocator,
-                cluster.replica_times[replica_index].time(),
-                &cluster.storages[replica_index],
-                &cluster.replica_pools[replica_index],
-                .{
-                    .node_count = cluster.options.replica_count + cluster.options.standby_count,
-                    .pipeline_requests_limit = cluster.replica_pipeline_requests_limit,
-                    .aof = &cluster.aofs[replica_index],
-                    .aof_recovery = false,
-                    // TODO Test restarting with a higher storage limit.
-                    .storage_size_limit = cluster.options.storage_size_limit,
-                    .nonce = options.nonce,
-                    .state_machine_options = cluster.options.state_machine,
-                    .message_bus_options = .{ .network = cluster.network },
-                    .release = options.release,
-                    .release_client_min = release_client_min,
-                    .multiversion = replica_multiversion(replica),
-                    .test_context = cluster,
-                    .tracer = &cluster.replica_tracers[replica_index],
-                    .replicate_options = cluster.options.replicate_options,
-                    .commit_stall_probability = null,
-                },
-            );
-            assert(replica.cluster == cluster.options.cluster_id);
-            assert(replica.replica == replica_index);
-            assert(replica.replica_count == cluster.replica_count);
-            assert(replica.standby_count == cluster.standby_count);
-
-            replica.event_callback = on_replica_event;
-            cluster.network.link(replica.message_bus.process, &replica.message_bus);
-        }
-
-        fn replica_multiversion(replica_context: *Replica) Multiversion {
-            const vtable = struct {
-                fn releases_bundled(context: *anyopaque) vsr.ReleaseList {
-                    const replica: *Replica = @ptrCast(@alignCast(context));
-                    const cluster: *Cluster = @ptrCast(@alignCast(replica.test_context.?));
-                    return cluster.replica_releases_bundled[replica.replica];
-                }
-                fn release_execute(context: *anyopaque, release_next: vsr.Release) void {
-                    const replica: *Replica = @ptrCast(@alignCast(context));
-                    const cluster: *Cluster = @ptrCast(@alignCast(replica.test_context.?));
-                    cluster.replica_release_execute_soon(replica, release_next);
-                }
-                fn tick(_: *anyopaque) void {}
-            };
-
-            return .{
-                .context = replica_context,
-                .vtable = &.{
-                    .releases_bundled = vtable.releases_bundled,
-                    .release_execute = vtable.release_execute,
-                    .tick = vtable.tick,
-                },
-            };
-        }
-
-        fn replica_release_execute_soon(
-            cluster: *Cluster,
-            replica: *Replica,
-            release: vsr.Release,
-        ) void {
-            assert(replica.release.value != release.value);
-            assert(cluster.replica_upgrades[replica.replica] == null);
-
-            log.debug("{}: release_execute_soon: release={}..{}", .{
-                replica.replica,
-                replica.release,
-                release,
-            });
-
-            if (cluster.replica_health[replica.replica] == .up) {
-                // The replica is trying to upgrade to a newer release at runtime.
-                assert(replica.journal.status != .init);
-                assert(replica.release.value < release.value);
-            } else {
-                assert(replica.journal.status == .init);
-                maybe(replica.release.value < release.value);
-            }
-
-            cluster.storages[replica.replica].reset();
-            cluster.replica_upgrades[replica.replica] = release;
-        }
-
-        /// `replica_upgrades` defers upgrades to the next tick (rather than executing it
-        /// immediately in replica_release_execute_soon()). Since we don't actually exec() to a new
-        /// version, this allows the replica to clean up properly (e.g. release Message's via
-        /// `defer`).
-        fn replica_release_execute(cluster: *Cluster, replica_index: u8) void {
-            const replica = &cluster.replicas[replica_index];
-            assert(cluster.replica_health[replica_index] == .up);
-
-            const release = cluster.replica_upgrades[replica_index].?;
-            defer cluster.replica_upgrades[replica_index] = null;
-
-            log.debug("{}: release_execute: release={}..{}", .{
-                replica_index,
-                replica.release,
-                release,
-            });
-
-            cluster.replica_crash(replica_index);
-
-            if (replica.multiversion.releases_bundled().contains(release)) {
-                // Disable faults while restarting to ensure that the cluster doesn't get stuck due
-                // to too many replicas in status=recovering_head.
-                const faulty = cluster.storages[replica_index].faulty;
-                cluster.storages[replica_index].faulty = false;
-                defer cluster.storages[replica_index].faulty = faulty;
-
-                cluster.replica_open(replica_index, .{
-                    .nonce = cluster.replicas[replica_index].nonce + 1,
-                    .release = release,
-                }) catch |err| {
-                    log.err("{}: release_execute failed: error={}", .{ replica_index, err });
-                    @panic("release_execute failed");
-                };
-                cluster.replica_enable(replica_index);
-            } else {
-                // The cluster has upgraded to `release`, but this replica does not have that
-                // release available yet.
-                log.debug("{}: release_execute: target version not available", .{replica_index});
-                assert(cluster.replica_health[replica_index] == .down);
-            }
-        }
-
-        pub fn replica_reformat(
-            cluster: *Cluster,
-            replica_index: u8,
-        ) !void {
-            assert(cluster.reformat_count < cluster.options.reformats_max);
-            assert(cluster.replica_health[replica_index] == .down);
-            assert(cluster.replica_reformats[replica_index] == null);
-            assert(replica_index < cluster.options.replica_count);
-
-            cluster.replica_health[replica_index] = .reformatting;
-            cluster.log_replica(.reformat, replica_index);
-
-            const storage = &cluster.storages[replica_index];
-            const storage_options = storage.options;
-            storage.deinit(cluster.allocator);
-            storage.* = try Storage.init(cluster.allocator, storage_options);
-
-            const client_index = cluster.options.client_count + cluster.reformat_count;
-            cluster.reformat_count += 1;
-            cluster.replica_reformats[replica_index] = try ReplicaReformat.init(
-                cluster.allocator,
-                &cluster.clients[client_index].?,
-                storage,
-                .{
-                    .cluster = cluster.options.cluster_id,
-                    .release = cluster.options.releases[0].release,
-                    .replica = @intCast(replica_index),
-                    .replica_count = cluster.options.replica_count,
-                    .view = null,
-                },
-            );
-            cluster.replica_reformats[replica_index].?.start();
-        }
-
-        pub fn register(cluster: *Cluster, client_index: usize) void {
-            const client = &cluster.clients[client_index].?;
-            client.register(register_callback, undefined);
-        }
-
-        /// See request_callback().
-        fn register_callback(
-            user_data: u128,
-            result: *const vsr.RegisterResult,
-        ) void {
-            _ = user_data;
-            _ = result;
-        }
-
-        pub fn request(
-            cluster: *Cluster,
-            client_index: usize,
-            request_operation: StateMachine.Operation,
-            request_message: *Message,
-            request_body_size: usize,
-        ) void {
-            assert(cluster.client_eviction_reasons[client_index] == null);
-
-            const client = &cluster.clients[client_index].?;
-            const message = request_message.build(.request);
-
-            message.header.* = .{
-                .release = client.release,
-                .client = client.id,
-                .request = 0, // Set by client.raw_request.
-                .cluster = client.cluster,
-                .command = .request,
-                .operation = request_operation.to_vsr(),
-                .size = @intCast(@sizeOf(vsr.Header) + request_body_size),
-                .previous_request_latency = cluster.prng.int(u32),
-            };
-
-            client.raw_request(
-                request_callback,
-                undefined,
-                message,
-            );
-            assert(message.header.request != 0);
-        }
-
-        /// The `request_callback` is not used — Cluster uses `Client.on_reply_{context,callback}`
-        /// instead because:
-        /// - Cluster needs access to the request
-        /// - Cluster needs access to the reply message (not just the body)
-        ///
-        /// See `on_reply`.
-        fn request_callback(
-            user_data: u128,
-            operation: vsr.Operation,
-            timestamp: u64,
-            result: []u8,
-        ) void {
-            _ = user_data;
-            _ = operation;
-            _ = timestamp;
-            _ = result;
-        }
-
-        fn client_on_reply(
-            client: *Client,
-            request_message: *Message.Request,
-            reply_message: *Message.Reply,
-        ) void {
-            const cluster: *Cluster = @ptrCast(@alignCast(client.on_reply_context.?));
-            assert(reply_message.header.invalid() == null);
-            assert(reply_message.header.cluster == cluster.options.cluster_id);
-            assert(reply_message.header.client == client.id);
-            assert(reply_message.header.request == request_message.header.request);
-            assert(reply_message.header.command == .reply);
-            assert(reply_message.header.operation == request_message.header.operation);
-
-            const client_index =
-                cluster.client_id_permutation.decode(client.id) - client_id_permutation_shift;
-            assert(&cluster.clients[client_index].? == client);
-            assert(cluster.client_eviction_reasons[client_index] == null);
-
-            if (cluster.callbacks.on_client_reply) |on_client_reply| {
-                on_client_reply(cluster, client_index, request_message, reply_message);
-            }
-        }
-
-        fn cluster_on_eviction(cluster: *Cluster, client_id: u128) void {
-            cluster.state_checker.on_client_eviction(client_id);
-        }
-
-        fn client_on_eviction(client: *Client, eviction: *const Message.Eviction) void {
-            const cluster: *Cluster = @ptrCast(@alignCast(client.on_reply_context.?));
-            assert(eviction.header.invalid() == null);
-            assert(eviction.header.cluster == cluster.options.cluster_id);
-            assert(eviction.header.client == client.id);
-            assert(eviction.header.command == .eviction);
-
-            const client_index =
-                cluster.client_id_permutation.decode(client.id) - client_id_permutation_shift;
-            assert(&cluster.clients[client_index].? == client);
-            assert(cluster.client_eviction_reasons[client_index] == null);
-
-            cluster.client_eviction_reasons[client_index] = eviction.header.reason;
-            cluster.network.process_disable(.{ .client = client.id });
-
-            cluster.client_eviction_requests_cancelled +=
-                @intFromBool(client.request_inflight != null and
-                client.request_inflight.?.message.header.operation != .register and
-                client.request_inflight.?.message.header.operation != .noop);
-        }
-
-        fn on_replica_event(replica: *const Replica, event: vsr.ReplicaEvent) void {
-            const cluster: *Cluster = @ptrCast(@alignCast(replica.test_context.?));
-            assert(cluster.replica_health[replica.replica] == .up);
-
-            switch (event) {
-                .message_sent => |message| {
-                    cluster.state_checker.on_message(message);
-                },
-                .state_machine_opened => {
-                    cluster.manifest_checker.forest_open(&replica.state_machine.forest);
-                },
-                .committed => |data| {
-                    assert(data.reply.header.client == data.prepare.header.client);
-
-                    cluster.log_replica(.commit, replica.replica);
-                    cluster.state_checker.check_state(replica.replica) catch |err| {
-                        fatal(.correctness, "state checker error: {}", .{err});
-                    };
-
-                    if (cluster.callbacks.on_cluster_reply) |on_cluster_reply| {
-                        const client_index = if (data.prepare.header.client == 0)
-                            null
-                        else
-                            cluster.client_id_permutation.decode(data.prepare.header.client) -
-                                client_id_permutation_shift;
-                        on_cluster_reply(cluster, client_index, data.prepare, data.reply);
-                    }
-                },
-                .compaction_completed => {
-                    cluster.storage_checker.replica_compact(Replica, replica) catch |err| {
-                        fatal(.correctness, "storage checker error: {}", .{err});
-                    };
-                },
-                .checkpoint_commenced => {
-                    cluster.log_replica(.checkpoint_commenced, replica.replica);
-                },
-                .checkpoint_completed => {
-                    cluster.log_replica(.checkpoint_completed, replica.replica);
-                    cluster.manifest_checker.forest_checkpoint(&replica.state_machine.forest);
-                    cluster.storage_checker.replica_checkpoint(Replica, replica) catch |err| {
-                        fatal(.correctness, "storage checker error: {}", .{err});
-                    };
-                },
-                .sync_stage_changed => switch (replica.syncing) {
-                    .idle => cluster.log_replica(.sync, replica.replica),
-                    .updating_checkpoint => {
-                        cluster.state_checker.check_state(replica.replica) catch |err| {
-                            fatal(.correctness, "state checker error: {}", .{err});
-                        };
-                    },
-                    else => {},
-                },
-                .client_evicted => |client_id| cluster.cluster_on_eviction(client_id),
-            }
-        }
-
-        /// Print an error message and then exit with an exit code.
-        fn fatal(failure: Failure, comptime fmt_string: []const u8, args: anytype) noreturn {
-            std.log.scoped(.state_checker).err(fmt_string, args);
-            std.posix.exit(@intFromEnum(failure));
-        }
-
-        /// Print the current state of the cluster, intended for printf debugging.
-        pub fn log_cluster(cluster: *const Cluster) void {
-            var replica: u8 = 0;
-            while (replica < cluster.replicas.len) : (replica += 1) {
-                cluster.log_replica(.commit, replica);
-            }
-        }
-
-        fn log_replica(
-            cluster: *const Cluster,
-            event: enum(u8) {
-                crash = '!',
-                recover = '^',
-                reformat = 'X',
-                commit = ' ',
-                sync = '$',
-                checkpoint_commenced = '[',
-                checkpoint_completed = ']',
-            },
-            replica_index: u8,
-        ) void {
-            const replica = &cluster.replicas[replica_index];
-
-            var statuses: [constants.members_max]u8 = @splat(' ');
-            statuses[replica_index] = switch (cluster.replica_health[replica_index]) {
-                .reformatting => ' ',
-                .down => '#',
-                .up => switch (replica.status) {
-                    .normal => @as(u8, '.'),
-                    .view_change => @as(u8, 'v'),
-                    .recovering => @as(u8, 'r'),
-                    .recovering_head => @as(u8, 'h'),
-                },
-            };
-
-            const role: u8 = role: {
-                if (cluster.replica_health[replica_index] == .down) break :role '#';
-                if (cluster.replica_health[replica_index] == .reformatting) break :role 'F';
-                if (replica.syncing != .idle) break :role '~';
-                if (replica.standby()) break :role '|';
-                if (replica.primary_index(replica.view) == replica.replica) break :role '/';
-                break :role '\\';
-            };
-
-            var info_buffer: [128]u8 = undefined;
-            var info: []u8 = "";
-            var pipeline_buffer: [16]u8 = undefined;
-            var pipeline: []u8 = "";
-
-            if (cluster.replica_health[replica_index] == .up) {
-                var journal_op_min: u64 = std.math.maxInt(u64);
-                var journal_op_max: u64 = 0;
-                if (replica.journal.status == .init) {
-                    // `journal.headers` is junk data when we are upgrading from Replica.open().
-                    assert(event == .recover);
-                    assert(cluster.replica_upgrades[replica_index] != null);
-                    journal_op_min = 0;
-                } else {
-                    for (replica.journal.headers) |*header| {
-                        if (header.operation != .reserved) {
-                            if (journal_op_min > header.op) journal_op_min = header.op;
-                            if (journal_op_max < header.op) journal_op_max = header.op;
-                        }
-                    }
-                }
-
-                var wal_op_min: u64 = std.math.maxInt(u64);
-                var wal_op_max: u64 = 0;
-                for (cluster.storages[replica_index].wal_prepares()) |*prepare| {
-                    if (prepare.header.valid_checksum() and
-                        prepare.header.command == .prepare)
-                    {
-                        if (wal_op_min > prepare.header.op) wal_op_min = prepare.header.op;
-                        if (wal_op_max < prepare.header.op) wal_op_max = prepare.header.op;
-                    }
-                }
-
-                info = std.fmt.bufPrint(&info_buffer, "" ++
-                    "{[view]:>4}V " ++
-                    "{[op_checkpoint]:>3}/{[commit_min]:_>3}/{[commit_max]:_>3}C " ++
-                    "{[journal_op_min]:>3}:{[journal_op_max]:_>3}Jo " ++
-                    "{[journal_faulty]:>2}/{[journal_dirty]:_>2}J! " ++
-                    "{[wal_op_min]:>3}:{[wal_op_max]:_>3}Wo " ++
-                    "<{[sync_op_min]:_>3}:{[sync_op_max]:_>3}> " ++
-                    "v{[release]}:{[release_max]} " ++
-                    "{[grid_blocks_acquired]?:>5}Ga " ++
-                    "{[grid_blocks_global]:>2}G! " ++
-                    "{[grid_blocks_repair]:>3}G?", .{
-                    .view = replica.view,
-                    .op_checkpoint = replica.op_checkpoint(),
-                    .commit_min = replica.commit_min,
-                    .commit_max = replica.commit_max,
-                    .journal_op_min = journal_op_min,
-                    .journal_op_max = journal_op_max,
-                    .journal_dirty = replica.journal.dirty.count,
-                    .journal_faulty = replica.journal.faulty.count,
-                    .wal_op_min = wal_op_min,
-                    .wal_op_max = wal_op_max,
-                    .sync_op_min = replica.superblock.working.vsr_state.sync_op_min,
-                    .sync_op_max = replica.superblock.working.vsr_state.sync_op_max,
-                    .release = replica.release.triple().patch,
-                    .release_max = replica.multiversion.releases_bundled().last().triple().patch,
-                    .grid_blocks_acquired = if (replica.grid.free_set.opened)
-                        replica.grid.free_set.count_acquired()
-                    else
-                        null,
-                    .grid_blocks_global = replica.grid.read_global_queue.count(),
-                    .grid_blocks_repair = replica.grid.blocks_missing.faulty_blocks.count(),
-                }) catch unreachable;
-
-                if (replica.pipeline == .queue) {
-                    pipeline = std.fmt.bufPrint(&pipeline_buffer, "  {:>2}/{}Pp {:>2}/{}Rq", .{
-                        replica.pipeline.queue.prepare_queue.count,
-                        constants.pipeline_prepare_queue_max,
-                        replica.pipeline.queue.request_queue.count,
-                        constants.pipeline_request_queue_max,
-                    }) catch unreachable;
-                }
-            }
-
-            log.info("{[replica]: >2} {[event]c} {[role]c} {[statuses]s}" ++
-                "  {[info]s}{[pipeline]s}", .{
-                .replica = replica.replica,
-                .event = @intFromEnum(event),
-                .role = role,
-                .statuses = statuses[0 .. cluster.replica_count + cluster.standby_count],
-                .info = info,
-                .pipeline = pipeline,
-            });
-        }
-    };
-}