tigerbeetle 0.0.40 → 0.17.8

data/ext/tb_client/tigerbeetle/src/vopr.zig

@@ -1,1785 +0,0 @@
-const std = @import("std");
-const stdx = @import("stdx");
-const builtin = @import("builtin");
-const assert = std.debug.assert;
-const maybe = stdx.maybe;
-const mem = std.mem;
-const ratio = stdx.PRNG.ratio;
-const Ratio = stdx.PRNG.Ratio;
-const range_inclusive_ms = @import("./testing/fuzz.zig").range_inclusive_ms;
-
-const constants = @import("constants.zig");
-const schema = @import("lsm/schema.zig");
-const vsr = @import("vsr.zig");
-const fuzz = @import("./testing/fuzz.zig");
-const Header = vsr.Header;
-
-pub const vsr_options = .{
-    .config_verify = true,
-    .git_commit = @import("vsr_options").git_commit,
-    .release = @import("vsr_options").release,
-    .release_client_min = @import("vsr_options").release_client_min,
-};
-const vsr_vopr_options = @import("vsr_vopr_options");
-
-const state_machine = vsr_vopr_options.state_machine;
-const StateMachineType = switch (state_machine) {
-    .accounting => @import("state_machine.zig").StateMachineType,
-    .testing => @import("testing/state_machine.zig").StateMachineType,
-};
-
-const Cluster = @import("testing/cluster.zig").ClusterType(StateMachineType);
-const Release = @import("testing/cluster.zig").Release;
-const StateMachine = Cluster.StateMachine;
-const Failure = @import("testing/cluster.zig").Failure;
-const PartitionMode = @import("testing/packet_simulator.zig").PartitionMode;
-const PartitionSymmetry = @import("testing/packet_simulator.zig").PartitionSymmetry;
-const Core = @import("testing/cluster/network.zig").Network.Core;
-const ReplySequence = @import("testing/reply_sequence.zig").ReplySequence;
-const Message = @import("message_pool.zig").MessagePool.Message;
-
-const MiB = stdx.MiB;
-
-const releases = [_]Release{
-    .{
-        .release = vsr.Release.from(.{ .major = 0, .minor = 0, .patch = 1 }),
-        .release_client_min = vsr.Release.from(.{ .major = 0, .minor = 0, .patch = 1 }),
-    },
-    .{
-        .release = vsr.Release.from(.{ .major = 0, .minor = 0, .patch = 2 }),
-        .release_client_min = vsr.Release.from(.{ .major = 0, .minor = 0, .patch = 1 }),
-    },
-    .{
-        .release = vsr.Release.from(.{ .major = 0, .minor = 0, .patch = 3 }),
-        .release_client_min = vsr.Release.from(.{ .major = 0, .minor = 0, .patch = 1 }),
-    },
-};
-
-const log = std.log.scoped(.simulator);
-
-pub const std_options: std.Options = .{
-    // The -vopr-log=<full|short> build option selects two logging modes.
-    // In "short" mode, only state transitions are printed (see `Cluster.log_replica`).
-    // "full" mode is the usual logging according to the level.
-    .log_level = if (vsr_vopr_options.log == .short) .info else .debug,
-    .logFn = log_override,
-
-    // Uncomment if you need per-scope control over the log levels.
-    // pub const log_scope_levels: []const std.log.ScopeLevel = &.{
-    //     .{ .scope = .cluster, .level = .info },
-    //     .{ .scope = .replica, .level = .debug },
-    // };
-};
-
-pub const tigerbeetle_config = @import("config.zig").configs.test_min;
-
-const cluster_id = 0;
-
-const CLIArgs = struct {
-    // "lite" mode runs a small cluster and only looks for crashes.
-    lite: bool = false,
-    performance: bool = false,
-    // Feel free to add more runtime overrides here!
-    ticks_max_requests: u32 = 40_000_000,
-    ticks_max_convergence: u32 = 10_000_000,
-    packet_loss_ratio: ?Ratio = null,
-    replica_missing: ?u8 = null,
-    replica_missing_until_request: ?u32 = null,
-    requests_max: ?u32 = null,
-
-    @"--": void,
-    seed: ?[]const u8 = null,
-};
-
-pub fn main() !void {
-    comptime assert(constants.verify);
-    // This must be initialized at runtime as stderr is not comptime known on e.g. Windows.
-    log_buffer.unbuffered_writer = std.io.getStdErr().writer();
-    fuzz.limit_ram();
-
-    var gpa_instance: std.heap.GeneralPurposeAllocator(.{}) = .{};
-    defer {
-        _ = gpa_instance.detectLeaks();
-        switch (gpa_instance.deinit()) {
-            .ok => {},
-            .leak => @panic("memory leaked"),
-        }
-    }
-
-    const gpa = gpa_instance.allocator();
-
-    var args = try std.process.argsWithAllocator(gpa);
-    defer args.deinit();
-
-    const cli_args = stdx.flags(&args, CLIArgs);
-    if (cli_args.lite and cli_args.performance) {
-        return vsr.fatal(.cli, "--lite and --performance are mutually exclusive", .{});
-    }
-    if (cli_args.replica_missing != null and !cli_args.performance) {
-        return vsr.fatal(.cli, "--replica-missing requires --performance", .{});
-    }
-    if (cli_args.replica_missing == null and cli_args.replica_missing_until_request != null) {
-        return vsr.fatal(.cli, "--replica-missing-until-request requires --replica-missing", .{});
-    }
-
-    log_performance_mode = cli_args.performance;
-
-    const seed_random = std.crypto.random.int(u64);
-    const seed = seed_from_arg: {
-        const seed_argument = cli_args.seed orelse break :seed_from_arg seed_random;
-        break :seed_from_arg vsr.testing.parse_seed(seed_argument);
-    };
-
-    // We do not support ReleaseFast or ReleaseSmall because they disable assertions.
-    comptime assert(builtin.mode == .Debug or builtin.mode == .ReleaseSafe);
-
-    if (seed == seed_random) {
-        if (builtin.mode != .ReleaseSafe) {
-            // If no seed is provided, than Debug is too slow and ReleaseSafe is much faster.
-            return vsr.fatal(
-                .cli,
-                "no seed provided: the simulator must be run with -OReleaseSafe",
-                .{},
-            );
-        }
-        if (vsr_vopr_options.log != .short) {
-            log.warn("no seed provided: full debug logs are enabled, this will be slow", .{});
-        }
-    }
-
-    var prng = stdx.PRNG.from_seed(seed);
-
-    var options = if (cli_args.lite)
-        options_lite(&prng)
-    else if (cli_args.performance)
-        options_performance(&prng)
-    else
-        options_swarm(&prng);
-
-    options.replica_missing = cli_args.replica_missing;
-    options.replica_missing_until_request = cli_args.replica_missing_until_request;
-    if (cli_args.packet_loss_ratio) |packet_loss_ratio| {
-        options.network.packet_loss_probability = packet_loss_ratio;
-    }
-    if (cli_args.requests_max) |requests_max| {
-        options.requests_max = requests_max;
-    }
-
-    log.info(
-        \\
-        \\          SEED={}
-        \\
-        \\          replicas={}
-        \\          standbys={}
-        \\          clients={}
-        \\          request_probability={}
-        \\          idle_on_probability={}
-        \\          idle_off_probability={}
-        \\          one_way_delay_mean={}
-        \\          one_way_delay_min={}
-        \\          packet_loss_probability={}
-        \\          path_maximum_capacity={} messages
-        \\          path_clog_duration_mean={}
-        \\          path_clog_probability={}
-        \\          packet_replay_probability={}
-        \\          partition_mode={s}
-        \\          partition_symmetry={s}
-        \\          partition_probability={}
-        \\          unpartition_probability={}
-        \\          partition_stability={} ticks
-        \\          unpartition_stability={} ticks
-        \\          read_latency_min={}
-        \\          read_latency_mean={}
-        \\          write_latency_min={}
-        \\          write_latency_mean={}
-        \\          read_fault_probability={}
-        \\          write_fault_probability={}
-        \\          crash_probability={}
-        \\          crash_stability={} ticks
-        \\          restart_probability={}
-        \\          restart_stability={} ticks
-    , .{
-        seed,
-        options.cluster.replica_count,
-        options.cluster.standby_count,
-        options.cluster.client_count,
-        options.request_probability,
-        options.request_idle_on_probability,
-        options.request_idle_off_probability,
-        options.network.one_way_delay_mean,
-        options.network.one_way_delay_min,
-        options.network.packet_loss_probability,
-        options.network.path_maximum_capacity,
-        options.network.path_clog_duration_mean,
-        options.network.path_clog_probability,
-        options.network.packet_replay_probability,
-        @tagName(options.network.partition_mode),
-        @tagName(options.network.partition_symmetry),
-        options.network.partition_probability,
-        options.network.unpartition_probability,
-        options.network.partition_stability,
-        options.network.unpartition_stability,
-        options.storage.read_latency_min,
-        options.storage.read_latency_mean,
-        options.storage.write_latency_min,
-        options.storage.write_latency_mean,
-        options.storage.read_fault_probability,
-        options.storage.write_fault_probability,
-        options.replica_crash_probability,
-        options.replica_crash_stability,
-        options.replica_restart_probability,
-        options.replica_restart_stability,
-    });
-
-    var simulator = try Simulator.init(gpa, &prng, options);
-    defer simulator.deinit(gpa);
-
-    if (cli_args.performance) {
-        // Simulate a missing replica by crashing it with ∞ stability.
-        if (options.replica_missing) |replica_index| {
-            if (replica_index > options.network.node_count) {
-                vsr.fatal(.cli, "--replica-index too large", .{});
-            }
-            simulator.cluster.replica_crash(replica_index);
-            simulator.replica_crash_stability[replica_index] = std.math.maxInt(u32);
-        }
-
-        // Warm-up the cluster before performance testing to get past the initial view change.
-        simulator.options.request_probability = Ratio.zero();
-        for (0..500) |_| simulator.tick();
-        simulator.options.request_probability = options.request_probability;
-    }
-
-    for (0..options.cluster.client_count) |client_index| {
-        simulator.cluster.register(client_index);
-    }
-
-    // Safety: replicas crash and restart; at any given point in time arbitrarily many replicas may
-    // be crashed, but each replica restarts eventually. The cluster must process all requests
-    // without split-brain.
-    var tick_total: u64 = 0;
-    var tick: u64 = 0;
-    var requests_done: bool = false;
-    var upgrades_done: bool = false;
-
-    while (tick < cli_args.ticks_max_requests) : (tick += 1) {
-        const requests_replied_old = simulator.requests_replied;
-        simulator.tick();
-        tick_total += 1;
-        if (simulator.requests_replied > requests_replied_old) {
-            tick = 0;
-        }
-        requests_done = simulator.requests_replied == simulator.options.requests_max;
-        upgrades_done =
-            for (simulator.cluster.replicas, simulator.cluster.replica_health) |*replica, health| {
-                if (health != .up) continue;
-                const release_latest = releases[simulator.replica_releases_limit - 1].release;
-                if (replica.release.value == release_latest.value) {
-                    break true;
-                }
-            } else false;
-
-        if (requests_done and upgrades_done) break;
-    }
-
-    if (cli_args.lite or
-        (cli_args.performance and cli_args.replica_missing_until_request == null))
-    {
-        // Don't care about convergence.
-    } else {
-        const core = if (requests_done and upgrades_done)
-            // Liveness: a core set of replicas is up and fully connected. The rest of the replicas
-            // might be crashed or partitioned permanently. The core should converge to the same
-            // state.
-            random_core(
-                simulator.prng,
-                simulator.options.cluster.replica_count,
-                simulator.options.cluster.standby_count,
-            )
-        else
-            // Safety mode ran out of ticks without completing its requests, so now we check whether
-            // it was correct to do so.
-            //
-            // Run a fully-connected core of replicas to repair all faulty grid blocks, headers, and
-            // prepares that can be repaired. Thereafter, only correlated faults should remain.
-            full_core(
-                simulator.options.cluster.replica_count,
-                simulator.options.cluster.standby_count,
-            );
-
-        simulator.transition_to_liveness_mode(core);
-
-        tick = 0;
-        while (tick < cli_args.ticks_max_convergence) : (tick += 1) {
-            simulator.tick();
-            tick_total += 1;
-            if (simulator.pending() == null) {
-                break;
-            }
-        }
-
-        if (simulator.pending()) |reason| {
-            if (try simulator.cluster_recoverable(gpa)) {
-                log.info("no liveness, final cluster state (core={b}):", .{simulator.core.bits});
-                simulator.cluster.log_cluster();
-                log.err("you can reproduce this failure with seed={}", .{seed});
-                fatal(.liveness, "no state convergence: {s}", .{reason});
-            }
-        } else {
-            const commits = simulator.cluster.state_checker.commits.items;
-            const last_checksum = commits[commits.len - 1].header.checksum;
-            for (simulator.cluster.aofs, 0..) |*aof, replica_index| {
-                if (simulator.core.is_set(replica_index)) {
-                    try aof.validate(gpa, last_checksum);
-                } else {
-                    try aof.validate(gpa, null);
-                }
-            }
-        }
-    }
-
-    if (cli_args.performance) {
-        log.info("\nMessages:\n{}", .{simulator.cluster.network.message_summary});
-    } else {
-        log.debug("\nMessages:\n{}", .{simulator.cluster.network.message_summary});
-    }
-
-    log.info("\n          PASSED ({} ticks)", .{tick_total});
-}
-
-fn options_swarm(prng: *stdx.PRNG) Simulator.Options {
-    const replica_count = prng.range_inclusive(u8, 1, constants.replicas_max);
-    const standby_count = prng.int_inclusive(u8, constants.standbys_max);
-    const node_count = replica_count + standby_count;
-    // -1 since otherwise it is possible that all clients will evict each other.
-    // (Due to retried register messages from the first set of evicted clients.
-    // See the "Cluster: eviction: session_too_low" replica test for a related scenario.)
-    const client_count = prng.range_inclusive(u8, 1, constants.clients_max * 2 - 1);
-
-    const batch_size_limit_min = comptime batch_size_limit_min: {
-        var event_size_max: u32 = @sizeOf(vsr.RegisterRequest);
-        for (std.enums.values(StateMachine.Operation)) |operation| {
-            event_size_max = @max(event_size_max, operation.event_size());
-        }
-        break :batch_size_limit_min event_size_max;
-    };
-    const batch_size_limit: u32 = if (prng.boolean())
-        constants.message_body_size_max
-    else
-        prng.range_inclusive(u32, batch_size_limit_min, constants.message_body_size_max);
-
-    const multi_batch_per_request_limit: u32 = multi_batch_per_request_limit: {
-        const event_max = @divFloor(batch_size_limit, batch_size_limit_min);
-        assert(event_max > 0);
-        break :multi_batch_per_request_limit if (event_max == 1) 1 else prng.range_inclusive(
-            u32,
-            1,
-            event_max - 1, // Minus one for the multi-batch trailer.
-        );
-    };
-
-    const storage_size_limit = vsr.sector_floor(
-        200 * MiB - prng.int_inclusive(u64, 20 * MiB),
-    );
-
-    const cluster_options: Cluster.Options = .{
-        .cluster_id = cluster_id,
-        .replica_count = replica_count,
-        .standby_count = standby_count,
-        .client_count = client_count,
-        .storage_size_limit = storage_size_limit,
-        .seed = prng.int(u64),
-        .releases = &releases,
-        .client_release = releases[0].release,
-        .reformats_max = replica_count + 2, // Arbitrary reformat limit.
-
-        .state_machine = switch (state_machine) {
-            .testing => .{
-                .batch_size_limit = batch_size_limit,
-                .lsm_forest_node_count = 4096,
-            },
-            .accounting => .{
-                .batch_size_limit = batch_size_limit,
-                .lsm_forest_compaction_block_count = prng.int_inclusive(u32, 256) +
-                    StateMachine.Forest.Options.compaction_block_count_min,
-                .lsm_forest_node_count = 4096,
-                .cache_entries_accounts = if (prng.boolean()) 256 else 0,
-                .cache_entries_transfers = if (prng.boolean()) 256 else 0,
-                .cache_entries_transfers_pending = if (prng.boolean()) 256 else 0,
-                .log_trace = true,
-                .aof_recovery = false,
-            },
-        },
-        .replicate_options = .{
-            .closed_loop = prng.chance(ratio(1, 5)),
-            .star = prng.chance(ratio(1, 5)),
-        },
-    };
-
-    const network_options: Cluster.NetworkOptions = .{
-        .node_count = node_count,
-        .client_count = client_count,
-
-        .seed = prng.int(u64),
-
-        .one_way_delay_min = range_inclusive_ms(prng, 0, 30),
-        .one_way_delay_mean = range_inclusive_ms(prng, 30, 100),
-        .packet_loss_probability = ratio(prng.int_inclusive(u8, 30), 100),
-        .path_maximum_capacity = prng.range_inclusive(u8, 2, 20),
-        .path_clog_duration_mean = range_inclusive_ms(prng, 0, 5_000),
-        .path_clog_probability = ratio(prng.int_inclusive(u8, 2), 100),
-        .packet_replay_probability = ratio(prng.int_inclusive(u8, 50), 100),
-
-        .partition_mode = prng.enum_uniform(PartitionMode),
-        .partition_symmetry = prng.enum_uniform(PartitionSymmetry),
-        .partition_probability = ratio(prng.int_inclusive(u8, 3), 100),
-        .unpartition_probability = ratio(prng.range_inclusive(u8, 1, 10), 100),
-        .partition_stability = 100 + prng.int_inclusive(u32, 100),
-        .unpartition_stability = prng.int_inclusive(u32, 20),
-    };
-
-    const read_latency_min = range_inclusive_ms(prng, 0, 30);
-    const write_latency_min = range_inclusive_ms(prng, 0, 30);
-    const storage_options: Cluster.Storage.Options = .{
-        .size = cluster_options.storage_size_limit,
-        .seed = prng.int(u64),
-        .read_latency_min = read_latency_min,
-        .read_latency_mean = range_inclusive_ms(prng, read_latency_min, 100),
-        .write_latency_min = write_latency_min,
-        .write_latency_mean = range_inclusive_ms(prng, write_latency_min, 1_000),
-        .read_fault_probability = ratio(prng.range_inclusive(u8, 0, 10), 100),
-        .write_fault_probability = ratio(prng.range_inclusive(u8, 0, 10), 100),
-        .write_misdirect_probability = ratio(prng.range_inclusive(u8, 0, 10), 100),
-        .crash_fault_probability = ratio(prng.range_inclusive(u8, 80, 100), 100),
-    };
-    const storage_fault_atlas: Cluster.StorageFaultAtlas.Options = .{
-        .faulty_superblock = true,
-        .faulty_wal_headers = replica_count > 1,
-        .faulty_wal_prepares = replica_count > 1,
-        .faulty_client_replies = replica_count > 1,
-        // >2 instead of >1 because in R=2, a lagging replica may sync to the leading replica,
-        // but then the leading replica may have the only copy of a block in the cluster.
-        .faulty_grid = replica_count > 2,
-    };
-
-    const workload_options = StateMachine.Workload.Options.generate(prng, .{
-        .batch_size_limit = batch_size_limit,
-        .multi_batch_per_request_limit = multi_batch_per_request_limit,
-        .client_count = client_count,
-        // TODO(DJ) Once Workload no longer needs in_flight_max, make stalled_queue_capacity
-        // private. Also maybe make it dynamic (computed from the client_count instead of
-        // clients_max).
-        .in_flight_max = ReplySequence.stalled_queue_capacity *
-            multi_batch_per_request_limit,
-    });
-
-    return .{
-        .cluster = cluster_options,
-        .network = network_options,
-        .storage = storage_options,
-        .storage_fault_atlas = storage_fault_atlas,
-        .workload = workload_options,
-        // TODO Swarm testing: Test long+few crashes and short+many crashes separately.
-        .replica_crash_probability = ratio(2, 10_000_000),
-        .replica_crash_stability = prng.int_inclusive(u32, 1_000),
-        .replica_restart_probability = ratio(2, 1_000_000),
-        .replica_restart_stability = prng.int_inclusive(u32, 1_000),
-        .replica_reformat_probability = ratio(30, 100),
-
-        .replica_pause_probability = ratio(8, 10_000_000),
-        .replica_pause_stability = prng.int_inclusive(u32, 1_000),
-        .replica_unpause_probability = ratio(8, 1_000_000),
-        .replica_unpause_stability = prng.int_inclusive(u32, 1_000),
-
-        .replica_release_advance_probability = ratio(1, 1_000_000),
-        .replica_release_catchup_probability = ratio(1, 100_000),
-
-        .requests_max = constants.journal_slot_count * 3,
-        .request_probability = ratio(prng.range_inclusive(u8, 1, 100), 100),
-        .request_idle_on_probability = ratio(prng.range_inclusive(u8, 0, 20), 100),
-        .request_idle_off_probability = ratio(prng.range_inclusive(u8, 10, 20), 100),
-    };
-}
-
-fn options_lite(prng: *stdx.PRNG) Simulator.Options {
-    var base = options_swarm(prng);
-    base.cluster.replica_count = 3;
-    base.cluster.standby_count = 0;
-    base.network.node_count = 3;
-    return base;
-}
-
-fn options_performance(prng: *stdx.PRNG) Simulator.Options {
-    const cluster_options: Cluster.Options = .{
-        .cluster_id = cluster_id,
-        .replica_count = 6,
-        .standby_count = 0,
-        .client_count = 4,
-        .storage_size_limit = vsr.sector_floor(200 * MiB),
-        .seed = prng.int(u64),
-        .releases = releases[0..1],
-        .client_release = releases[0].release,
-        .reformats_max = 0,
-
-        .state_machine = switch (state_machine) {
-            .testing => .{
-                .batch_size_limit = constants.message_body_size_max,
-                .lsm_forest_node_count = 4096,
-            },
-            .accounting => .{
-                .batch_size_limit = constants.message_body_size_max,
-                .lsm_forest_compaction_block_count = 128 +
-                    StateMachine.Forest.Options.compaction_block_count_min,
-                .lsm_forest_node_count = 4096,
-                .cache_entries_accounts = 256,
-                .cache_entries_transfers = 0,
-                .cache_entries_transfers_pending = 0,
-                .log_trace = true,
-                .aof_recovery = false,
-            },
-        },
-    };
-
-    const network_options: Cluster.NetworkOptions = .{
-        .node_count = cluster_options.replica_count,
-        .client_count = cluster_options.client_count,
-
-        .seed = prng.int(u64),
-
-        .one_way_delay_mean = .ms(50),
-        .one_way_delay_min = .{ .ns = 0 },
-        .packet_loss_probability = Ratio.zero(),
-        .path_maximum_capacity = 10,
-        .path_clog_duration_mean = .ms(2_000),
-        .path_clog_probability = Ratio.zero(),
-        .packet_replay_probability = Ratio.zero(),
-
-        .partition_mode = .none,
-        .partition_symmetry = .symmetric,
-        .partition_probability = Ratio.zero(),
-        .unpartition_probability = Ratio.zero(),
-        .partition_stability = 100,
-        .unpartition_stability = 10,
-    };
-
-    const storage_options: Cluster.Storage.Options = .{
-        .size = cluster_options.storage_size_limit,
-        .seed = prng.int(u64),
-        .read_latency_min = .{ .ns = 0 },
-        .read_latency_mean = .{ .ns = 0 },
-        .write_latency_min = .{ .ns = 0 },
-        .write_latency_mean = .{ .ns = 0 },
-        .read_fault_probability = Ratio.zero(),
-        .write_fault_probability = Ratio.zero(),
-        .write_misdirect_probability = Ratio.zero(),
-        .crash_fault_probability = Ratio.zero(),
-    };
-    const storage_fault_atlas: Cluster.StorageFaultAtlas.Options = .{
-        .faulty_superblock = false,
-        .faulty_wal_headers = false,
-        .faulty_wal_prepares = false,
-        .faulty_client_replies = false,
-        .faulty_grid = false,
-    };
-
-    var workload_prng = stdx.PRNG.from_seed(92); // Fix workload for perf testing.
-    const workload_options = StateMachine.Workload.Options.generate(&workload_prng, .{
-        .batch_size_limit = constants.message_body_size_max,
-        .multi_batch_per_request_limit = 1,
-        .client_count = cluster_options.client_count,
-        .in_flight_max = ReplySequence.stalled_queue_capacity,
-    });
-
-    return .{
-        .cluster = cluster_options,
-        .network = network_options,
-        .storage = storage_options,
-        .storage_fault_atlas = storage_fault_atlas,
-        .workload = workload_options,
-        .replica_crash_probability = Ratio.zero(),
-        .replica_crash_stability = 500,
-        .replica_restart_probability = Ratio.zero(),
-        .replica_restart_stability = 500,
-        .replica_reformat_probability = ratio(0, 100),
-
-        .replica_pause_probability = Ratio.zero(),
-        .replica_pause_stability = 500,
-        .replica_unpause_probability = Ratio.zero(),
-        .replica_unpause_stability = 500,
-
-        .replica_release_advance_probability = Ratio.zero(),
-        .replica_release_catchup_probability = Ratio.zero(),
-
-        .requests_max = constants.journal_slot_count * 8,
-        .request_probability = ratio(100, 100),
-        .request_idle_on_probability = Ratio.zero(),
-        .request_idle_off_probability = ratio(100, 100),
-    };
-}
-
-pub const Simulator = struct {
-    pub const Options = struct {
-        cluster: Cluster.Options,
-        network: Cluster.NetworkOptions,
-        storage: Cluster.Storage.Options,
-        storage_fault_atlas: Cluster.StorageFaultAtlas.Options,
-
-        workload: StateMachine.Workload.Options,
-
-        /// Probability per tick that a crash will occur.
-        replica_crash_probability: Ratio,
-        /// Minimum duration of a crash.
-        replica_crash_stability: u32,
-        /// Probability per tick that a crashed replica will recovery.
-        replica_restart_probability: Ratio,
-        /// Minimum time a replica is up until it is crashed again.
-        replica_restart_stability: u32,
-        /// Probability per restart that a replica will be reformatted with `tigerbeetle recover`
-        /// (immediately before being restarted).
-        replica_reformat_probability: Ratio,
-
-        // A replica permanently or temporarily missing from the cluster, used in performance mode.
-        replica_missing: ?u8 = null,
-        /// Restart `replica_missing` after the specified request has received its reply.
-        replica_missing_until_request: ?u32 = null,
-
-        replica_pause_probability: Ratio,
-        replica_pause_stability: u32,
-        replica_unpause_probability: Ratio,
-        replica_unpause_stability: u32,
-
-        /// Probability per tick that a healthy replica will be crash-upgraded.
-        /// This probability is set to 0 during liveness mode.
-        replica_release_advance_probability: Ratio,
-        /// Probability that a crashed with an outdated version will be upgraded as it restarts.
-        /// This helps ensure that when the cluster upgrades, that replicas without the newest
-        /// version don't take too long to receive that new version.
-        /// This probability is set to 0 during liveness mode.
-        replica_release_catchup_probability: Ratio,
-
-        /// The total number of requests to send. Does not count `register` messages.
-        requests_max: usize,
-        request_probability: Ratio,
-        request_idle_on_probability: Ratio,
-        request_idle_off_probability: Ratio,
-    };
-
-    prng: *stdx.PRNG,
-    options: Options,
-    cluster: *Cluster,
-    workload: StateMachine.Workload,
-
-    // The number of releases in each replica's "binary".
-    replica_releases: []usize,
-    /// The maximum number of releases available in any replica's "binary".
-    /// (i.e. the maximum of any `replica_releases`.)
-    replica_releases_limit: usize = 1,
-
-    /// Keep track of which replicas have possibly "lost" data.
-    // TODO We could unset this when a replica fully recovers.
-    replica_reformats: Core = .{},
-
-    /// Protect a replica from fast successive crash/restarts.
-    replica_crash_stability: []usize,
-    reply_sequence: ReplySequence,
-    reply_op_next: u64 = 1, // Skip the root op.
-
-    /// Fully-connected subgraph of replicas for liveness checking.
-    core: Core = .{},
-
-    /// Total number of requests sent, including those that have not been delivered.
-    /// Does not include `register` messages.
-    requests_sent: usize = 0,
-    /// Total number of replies received by non-evicted clients.
-    /// Does not include `register` messages.
-    requests_replied: usize = 0,
-    requests_idle: bool = false,
-
-    pub fn init(
-        gpa: std.mem.Allocator,
-        prng: *stdx.PRNG,
-        options: Options,
-    ) !Simulator {
-        assert(options.requests_max > 0);
-        assert(options.request_probability.numerator > 0);
-        assert(options.request_idle_off_probability.numerator > 0);
-
-        var cluster = try Cluster.init(gpa, .{
-            .cluster = options.cluster,
-            .network = options.network,
-            .storage = options.storage,
-            .storage_fault_atlas = options.storage_fault_atlas,
-            .callbacks = .{
-                .on_cluster_reply = on_cluster_reply,
-                .on_client_reply = on_client_reply,
-            },
-        });
-        errdefer cluster.deinit();
-
-        var workload = try StateMachine.Workload.init(gpa, prng, options.workload);
-        errdefer workload.deinit(gpa);
-
-        const replica_releases = try gpa.alloc(
-            usize,
-            options.cluster.replica_count + options.cluster.standby_count,
-        );
-        errdefer gpa.free(replica_releases);
-        @memset(replica_releases, 1);
-
-        const replica_crash_stability = try gpa.alloc(
-            usize,
-            options.cluster.replica_count + options.cluster.standby_count,
-        );
-        errdefer gpa.free(replica_crash_stability);
-        @memset(replica_crash_stability, 0);
-
-        var reply_sequence = try ReplySequence.init(gpa);
-        errdefer reply_sequence.deinit(gpa);
-
-        return Simulator{
-            .prng = prng,
-            .options = options,
-            .cluster = cluster,
-            .workload = workload,
-            .replica_releases = replica_releases,
-            .replica_crash_stability = replica_crash_stability,
-            .reply_sequence = reply_sequence,
-        };
-    }
-
-    pub fn deinit(simulator: *Simulator, gpa: std.mem.Allocator) void {
-        gpa.free(simulator.replica_releases);
-        gpa.free(simulator.replica_crash_stability);
-        simulator.reply_sequence.deinit(gpa);
-        simulator.workload.deinit(gpa);
-        simulator.cluster.deinit();
-    }
-
-    pub fn pending(simulator: *const Simulator) ?[]const u8 {
-        assert(simulator.core.count() > 0);
-        assert(simulator.requests_sent - simulator.cluster.client_eviction_requests_cancelled <=
-            simulator.options.requests_max);
-        assert(simulator.reply_sequence.empty());
-        for (simulator.cluster.clients) |*client_maybe| {
-            if (client_maybe.*) |client| {
-                if (client.request_inflight) |_| return "pending request";
-            }
-        }
-
-        // Even though there are no client requests in progress, the cluster may be upgrading.
-        const release_max = simulator.core_release_max();
-        for (simulator.cluster.replicas) |*replica| {
-            if (simulator.core.is_set(replica.replica)) {
-                // (If down, the replica is waiting to be upgraded.)
-                maybe(simulator.cluster.replica_health[replica.replica] == .down);
-
-                if (replica.release.value != release_max.value) return "pending upgrade";
-            }
-        }
-
-        for (simulator.cluster.replicas) |*replica| {
-            if (simulator.core.is_set(replica.replica)) {
-                if (!simulator.cluster.state_checker.replica_convergence(replica.replica)) {
-                    return "pending replica convergence";
-                }
-            }
-        }
-
-        simulator.cluster.state_checker.assert_cluster_convergence();
-
-        // Check whether the replica is still repairing prepares/tables/replies.
-        const commit_max: u64 = simulator.cluster.state_checker.commits.items.len - 1;
-        for (simulator.cluster.replicas) |*replica| {
-            if (simulator.core.is_set(replica.replica)) {
-                for (replica.op_checkpoint() + 1..commit_max + 1) |op| {
-                    const header = simulator.cluster.state_checker.header_with_op(op);
-                    if (!replica.journal.has_prepare(&header)) return "pending journal";
-                }
-                // It's okay for a replica to miss some prepares older than the current checkpoint.
-                maybe(replica.journal.faulty.count > 0);
-
-                if (!replica.sync_content_done()) return "pending sync content";
-            }
-        }
-
-        // Expect that all core replicas have arrived at an identical (non-divergent) checkpoint.
-        var checkpoint_id: ?u128 = null;
-        for (simulator.cluster.replicas) |*replica| {
-            if (simulator.core.is_set(replica.replica)) {
-                const replica_checkpoint_id = replica.superblock.working.checkpoint_id();
-                if (checkpoint_id) |id| {
-                    assert(checkpoint_id == id);
-                } else {
-                    checkpoint_id = replica_checkpoint_id;
-                }
-            }
-        }
-        assert(checkpoint_id != null);
-
-        return null;
-    }
-
-    pub fn tick(simulator: *Simulator) void {
-        // TODO(Zig): Remove (see on_cluster_reply()).
-        simulator.cluster.context = simulator;
-
-        simulator.cluster.tick();
-        simulator.tick_requests();
-        simulator.tick_upgrade();
-        simulator.tick_crash();
-        simulator.tick_pause();
-
-        if (simulator.options.replica_missing_until_request) |request| {
-            if (simulator.requests_replied >= request) {
-                simulator.options.replica_missing_until_request = null;
-                simulator.replica_crash_stability[simulator.options.replica_missing.?] = 1;
-            }
-        }
-    }
-
-    pub fn cluster_recoverable(simulator: *Simulator, gpa: std.mem.Allocator) !bool {
-        if (simulator.core_missing_primary()) {
-            unimplemented("repair requires reachable primary");
-        } else if (simulator.core_missing_quorum()) {
-            log.warn("no liveness, core replicas cannot view-change", .{});
-        } else if (try simulator.core_missing_prepare(gpa)) |op| {
-            log.warn("no liveness, op={} is not available in core", .{op});
-        } else if (try simulator.core_missing_blocks(gpa)) |blocks| {
-            log.warn("no liveness, {} blocks are not available in core", .{blocks});
-        } else if (simulator.core_missing_reply()) |header| {
-            log.warn("no liveness, reply op={} is not available in core", .{header.op});
-        } else if (simulator.core_reformat_evicted()) {
-            log.warn("no liveness, one or more reformat clients was evicted", .{});
-        } else {
-            return true;
-        }
-
-        return false;
-    }
-
-    /// Executes the following:
-    /// * Restart any core replicas that are down at the moment
-    /// * Heal all network partitions between core replicas
-    /// * Disable storage faults on the core replicas
-    /// * For all failures involving non-core replicas, make those failures permanent.
-    ///
-    /// See https://tigerbeetle.com/blog/2023-07-06-simulation-testing-for-liveness for broader
-    /// context.
-    pub fn transition_to_liveness_mode(simulator: *Simulator, core: Core) void {
-        log.debug("transition_to_liveness_mode: core={b}", .{core.bits});
-        assert(simulator.core.count() == 0);
-        defer assert(simulator.core.count() > 0);
-
-        simulator.core = core;
-
-        var it = core.iterate();
-        while (it.next()) |replica_index| {
-            const fault = false;
-            if (simulator.cluster.replica_health[replica_index] == .down) {
-                simulator.replica_restart(@intCast(replica_index), fault);
-            }
-
-            const replica_health = simulator.cluster.replica_health[replica_index];
-            if (replica_health == .up and replica_health.up.paused) {
-                simulator.cluster.replica_unpause(@intCast(replica_index));
-            }
-
-            simulator.cluster.storages[replica_index].transition_to_liveness_mode();
-        }
-
-        simulator.cluster.network.transition_to_liveness_mode(simulator.core);
-        simulator.options.replica_crash_probability = Ratio.zero();
-        simulator.options.replica_restart_probability = Ratio.zero();
-        simulator.options.replica_reformat_probability = Ratio.zero();
-        simulator.options.replica_pause_probability = Ratio.zero();
-        simulator.options.replica_release_advance_probability = Ratio.zero();
-        simulator.options.replica_release_catchup_probability = Ratio.zero();
-    }
-
-    // If a primary ends up being outside of a core, and is only partially connected to the core,
-    // the core might fail to converge, as parts of the repair protocol rely on primary-sent
-    // `.start_view_change` messages. Until we fix this issue, we special-case this scenario in
-    // VOPR and don't treat it as a liveness failure.
-    //
-    // TODO: make sure that .recovering_head replicas can transition to normal even without direct
-    // connection to the primary
-    pub fn core_missing_primary(simulator: *const Simulator) bool {
-        assert(simulator.core.count() > 0);
-
-        for (simulator.cluster.replicas) |*replica| {
-            if (simulator.cluster.replica_health[replica.replica] == .up and
-                replica.status == .normal and replica.primary() and
-                !simulator.core.is_set(replica.replica))
-            {
-                // `replica` considers itself a primary, check that at least part of the core thinks
-                // so as well.
-                var it = simulator.core.iterate();
-                while (it.next()) |replica_core_index| {
-                    if (simulator.cluster.replicas[replica_core_index].view == replica.view) {
-                        return true;
-                    }
-                }
-            }
-        }
-        return false;
-    }
-
-    /// The core contains at least a view-change quorum of replicas. But if one or more of those
-    /// replicas are in status=recovering_head (due to corruption) or are stuck reformatting, then
-    /// that may be insufficient.
-    pub fn core_missing_quorum(simulator: *const Simulator) bool {
-        assert(simulator.core.count() > 0);
-
-        var core_replicas: u8 = 0;
-        var core_recovering: u8 = 0;
-        for (
-            simulator.cluster.replicas,
-            simulator.cluster.replica_health,
-        ) |*replica, health| {
-            if (simulator.core.is_set(replica.replica) and !replica.standby()) {
-                core_replicas += 1;
-                switch (health) {
-                    .up => core_recovering += @intFromBool(replica.status == .recovering_head),
-                    .down => unreachable,
-                    .reformatting => core_recovering += 1,
-                }
-            }
-        }
-
-        const quorums = vsr.quorums(simulator.options.cluster.replica_count);
-        assert(quorums.view_change <= core_replicas);
-        return quorums.view_change > core_replicas - core_recovering;
-    }
-
-    fn core_repairable_replica(
-        simulator: *const Simulator,
-        comptime Replica: type,
-        replica: *const Replica,
-    ) bool {
-        if (!simulator.core.is_set(replica.replica)) return false;
-        if (replica.standby()) return false;
-        if (simulator.cluster.replica_health[replica.replica] == .reformatting) return false;
-        assert(simulator.cluster.replica_health[replica.replica] == .up);
-
-        switch (replica.status) {
-            .normal => return true,
-            .recovering_head => return false,
-            // Lagging replicas do not initiate WAL repair during view change.
-            .view_change => return !vsr.Checkpoint.durable(
-                replica.op_checkpoint_next(),
-                replica.commit_max,
-            ),
-            .recovering => unreachable,
-        }
-    }
-
-    // Returns an op for a prepare which can't be repaired by the core due to storage faults.
-    //
-    // If a replica cannot make progress on committing, then it may be stuck while repairing either
-    // missing headers *or* prepares (see `repair` in replica.zig). This function checks for both.
-    //
-    // When generating a FaultAtlas, we don't try to protect core from excessive errors. Instead,
-    // if the core gets stuck, we verify that this is indeed due to storage faults.
-    pub fn core_missing_prepare(
-        simulator: *const Simulator,
-        gpa: std.mem.Allocator,
-    ) error{OutOfMemory}!?u64 {
-        assert(simulator.core.count() > 0);
-        const replica_count = simulator.options.cluster.replica_count;
-
-        var cluster_op_head: u64 = 0;
-        var cluster_commit_max: u64 = 0;
-        var cluster_log_view: u32 = 0;
-        // maxInt(u16) is more than enough to accommodate `requests_max`.
-        var cluster_op_repair_min: u64 = std.math.maxInt(u16);
-
-        for (simulator.cluster.replicas) |replica| {
-            if (!simulator.core_repairable_replica(Cluster.Replica, &replica)) continue;
-
-            if (replica.log_view > cluster_log_view) {
-                maybe(cluster_op_head > replica.op);
-                cluster_op_head = replica.op;
-            } else if (replica.log_view == cluster_log_view) {
-                cluster_op_head = @max(cluster_op_head, replica.op);
-            }
-
-            cluster_log_view = @max(cluster_log_view, replica.log_view);
-            cluster_commit_max = @max(cluster_commit_max, replica.commit_max);
-            cluster_op_repair_min = @min(cluster_op_repair_min, replica.op_repair_min());
-        }
-        assert(cluster_commit_max <= cluster_op_head);
-
-        // Use replicas with the largest log_view to infer uncommitted headers. Replicas with a
-        // smaller log_view may have an outdated version of the same uncommitted op. There may be
-        // at most a pipeline of uncommitted headers in the cluster.
-        const pipeline_max = constants.pipeline_prepare_queue_max;
-        var uncommitted_headers: [pipeline_max]?vsr.Header.Prepare = @splat(null);
-        if (cluster_commit_max < cluster_op_head) {
-            for (simulator.cluster.replicas) |replica| {
-                if (!simulator.core_repairable_replica(Cluster.Replica, &replica)) continue;
-
-                if (replica.log_view < cluster_log_view) continue;
-                for (cluster_commit_max + 1..cluster_op_head + 1) |op| {
-                    if (header: {
-                        if (replica.superblock.working.vsr_state.log_view <
-                            replica.superblock.working.vsr_state.view)
-                        {
-                            // When we are view-changing, our journal headers may contain headers
-                            // which were truncated then restored due to restart. If a view change
-                            // completes then that will be resolved, but if the view-change is stuck
-                            // (e.g. due to "quorum received, awaiting repair") then they may
-                            // linger, so if we only looked at the journal headers it would appear
-                            // as if the replicas disagreed about the uncommitted headers.
-                            const headers_count = replica.superblock.working.view_headers_count;
-                            const headers =
-                                replica.superblock.working.view_headers_all[0..headers_count];
-                            for (headers) |*header| {
-                                if (header.op == op) {
-                                    break :header switch (vsr.Headers.dvc_header_type(header)) {
-                                        .valid => header,
-                                        .blank => null,
-                                    };
-                                }
-                            } else break :header null;
-                        } else {
-                            break :header replica.journal.header_with_op(op);
-                        }
-                    }) |header| {
-                        if (uncommitted_headers[op % pipeline_max]) |header_existing| {
-                            assert(header_existing.op == header.op);
-                            assert(header_existing.view == header.view);
-                            assert(header_existing.checksum == header.checksum);
-                        } else {
-                            uncommitted_headers[op % pipeline_max] = header.*;
-                        }
-                    }
-                }
-            }
-        }
-
-        for (cluster_op_repair_min..cluster_op_head + 1) |op| {
-            if (op > cluster_commit_max) {
-                if (uncommitted_headers[op % pipeline_max] == null) {
-                    // We can only be missing an uncommitted *header* (and be unable to nack it)
-                    // if at least one replica was reformatted.
-                    var core_replicas = simulator.core.iterate();
-                    while (core_replicas.next()) |replica| {
-                        if (simulator.replica_reformats.is_set(replica)) break;
-                    } else unreachable;
-
-                    return op;
-                }
-            }
-        }
-
-        const ReplicaSet = stdx.BitSetType(constants.replicas_max);
-        var replicas_missing_ops = try gpa.alloc(
-            ReplicaSet,
-            cluster_op_head - cluster_op_repair_min + 1,
-        );
-        defer gpa.free(replicas_missing_ops);
-
-        for (replicas_missing_ops, cluster_op_repair_min..) |*replicas_missing_op, op| {
-            replicas_missing_op.* = .{};
-            const header = blk: {
-                if (op > cluster_commit_max) {
-                    const uncommitted_header = uncommitted_headers[op % pipeline_max].?;
-                    assert(uncommitted_header.op == op);
-                    break :blk uncommitted_header;
-                } else {
-                    break :blk simulator.cluster.state_checker.header_with_op(op);
-                }
-            };
-            for (simulator.cluster.replicas) |replica| {
-                // Replicas should be able to repair using any other replica in the core.
-                if (replica.standby()) continue;
-                if (simulator.cluster.replica_health[replica.replica] == .reformatting or
-                    !simulator.core.is_set(replica.replica) or
-                    !replica.journal.has_prepare(&header))
-                {
-                    replicas_missing_op.set(replica.replica);
-                }
-            }
-        }
-
-        // Check whether any of the uncommitted headers is corrupted on more than a nack
-        // quorum of replicas. If so, the cluster cannot initiate repair or commit (see the
-        // awaiting_repair and complete_invalid cases in the DVCQuorum).
-        const nack_quorum = vsr.quorums(replica_count).nack_prepare;
-        for (cluster_commit_max..cluster_op_head + 1) |op| {
-            if (replicas_missing_ops[op - cluster_op_repair_min].count() >= nack_quorum) {
-                const header = blk: {
-                    if (op > cluster_commit_max) {
-                        const uncommitted_header = uncommitted_headers[op % pipeline_max].?;
-                        assert(uncommitted_header.op == op);
-                        break :blk uncommitted_header;
-                    } else {
-                        break :blk simulator.cluster.state_checker.header_with_op(op);
-                    }
-                };
-                return header.op;
-            }
-        }
-
-        for (simulator.cluster.replicas) |replica| {
-            if (!simulator.core_repairable_replica(Cluster.Replica, &replica)) continue;
-            if (simulator.cluster.replica_health[replica.replica] == .reformatting) continue;
-
-            // Check prepares between (commit_min, commit_max], replicas repair these first
-            // as commit progress depends on them.
-            if (replica.commit_min < replica.commit_max) {
-                for (replica.commit_min + 1..replica.commit_max + 1) |op| {
-                    if (replicas_missing_ops[op - cluster_op_repair_min].count() == replica_count) {
-                        return op;
-                    }
-                }
-            }
-
-            // Check prepares between [op_repair_min, commit_min] as view changing replicas
-            // cannot step up as primary unless they have all prepares intact.
-            if (replica.op_repair_min() <= replica.commit_min) {
-                for (replica.op_repair_min()..replica.commit_min + 1) |op| {
-                    if (replicas_missing_ops[op - cluster_op_repair_min].count() == replica_count) {
-                        return op;
-                    }
-                }
-            }
-        }
-        return null;
-    }
-
-    /// Check whether the cluster is stuck because the entire core is missing the same block[s].
-    pub fn core_missing_blocks(
-        simulator: *const Simulator,
-        gpa: std.mem.Allocator,
-    ) error{OutOfMemory}!?usize {
-        assert(simulator.core.count() > 0);
-
-        const FaultyReplicas = stdx.BitSetType(constants.members_max);
-        var blocks_missing = std.AutoArrayHashMap(
-            struct { address: u64, checksum: u128 },
-            FaultyReplicas,
-        ).init(gpa);
-        defer blocks_missing.deinit();
-
-        // Find all blocks that any replica in the core is missing.
-        for (simulator.cluster.replicas) |replica| {
-            if (!simulator.core.is_set(replica.replica)) continue;
-            if (simulator.cluster.replica_health[replica.replica] == .reformatting) continue;
-
-            const storage = &simulator.cluster.storages[replica.replica];
-
-            var fault_iterator = replica.grid.read_global_queue.iterate();
-            while (fault_iterator.next()) |faulty_read| {
-                const v = try blocks_missing.getOrPut(.{
-                    .address = faulty_read.address,
-                    .checksum = faulty_read.checksum,
-                });
-
-                if (!v.found_existing) v.value_ptr.* = .{};
-                v.value_ptr.set(replica.replica);
-
-                log.debug("{}: core_missing_blocks: " ++
-                    "missing address={} checksum={x:0>32} corrupt={} (remote read)", .{
-                    replica.replica,
-                    faulty_read.address,
-                    faulty_read.checksum,
-                    storage.area_faulty(.{ .grid = .{ .address = faulty_read.address } }),
-                });
-            }
-
-            var repair_iterator = replica.grid.blocks_missing.faulty_blocks.iterator();
-            while (repair_iterator.next()) |fault| {
-                const v = try blocks_missing.getOrPut(.{
-                    .address = fault.key_ptr.*,
-                    .checksum = fault.value_ptr.checksum,
-                });
-
-                if (!v.found_existing) v.value_ptr.* = .{};
-                v.value_ptr.set(replica.replica);
-
-                log.debug("{}: core_missing_blocks: " ++
-                    "missing address={} checksum={x:0>32} corrupt={} (GridBlocksMissing)", .{
-                    replica.replica,
-                    fault.key_ptr.*,
-                    fault.value_ptr.checksum,
-                    storage.area_faulty(.{ .grid = .{ .address = fault.key_ptr.* } }),
-                });
-            }
-        }
-
-        // Check whether every replica in the core is missing the blocks.
-        // (If any core replica has the block, then that is a bug, since it should have repaired.)
-        var blocks_missing_iterator = blocks_missing.iterator();
-        while (blocks_missing_iterator.next()) |block_missing_and_faulty_replicas| {
-            const block_missing = block_missing_and_faulty_replicas.key_ptr;
-            const faulty_replicas = block_missing_and_faulty_replicas.value_ptr;
-            for (simulator.cluster.replicas) |replica| {
-                const storage = &simulator.cluster.storages[replica.replica];
-
-                // A replica might actually have the block that it is requesting, but not know.
-                // This can occur after state sync: if we compact and create a table, but then skip
-                // over that table via state sync, we will try to sync the table anyway.
-                if (faulty_replicas.is_set(replica.replica)) continue;
-
-                if (!simulator.core.is_set(replica.replica)) continue;
-                if (simulator.cluster.replica_health[replica.replica] == .reformatting) continue;
-                if (replica.standby()) continue;
-                if (storage.area_faulty(.{
-                    .grid = .{ .address = block_missing.address },
-                })) continue;
-
-                const block = storage.grid_block(block_missing.address) orelse continue;
-                const block_header = schema.header_from_block(block);
-                if (block_header.checksum == block_missing.checksum) {
-                    log.err("{}: core_missing_blocks: found address={} checksum={x:0>32}", .{
-                        replica.replica,
-                        block_missing.address,
-                        block_missing.checksum,
-                    });
-                    @panic("block found in core");
-                }
-            }
-        }
-
-        if (blocks_missing.count() == 0) {
-            return null;
-        } else {
-            return blocks_missing.count();
-        }
-    }
-
-    /// Check whether the cluster is stuck because the entire core is missing the same reply[s].
-    pub fn core_missing_reply(simulator: *const Simulator) ?vsr.Header.Reply {
-        assert(simulator.core.count() > 0);
-
-        for (simulator.cluster.state_checker.client_replies.values()) |reply| {
-            const reply_in_core = reply_in_core: for (simulator.cluster.replicas) |replica| {
-                if (simulator.cluster.replica_health[replica.replica] == .reformatting) continue;
-
-                const storage = &simulator.cluster.storages[replica.replica];
-                const storage_replies = storage.client_replies();
-                if (simulator.core.is_set(replica.replica) and !replica.standby()) {
-                    for (storage_replies, 0..) |storage_reply, reply_slot| {
-                        if (storage_reply.header.checksum == reply.checksum and
-                            !storage.area_faulty(.{ .client_replies = .{ .slot = reply_slot } }))
-                        {
-                            break :reply_in_core true;
-                        }
-                    }
-                }
-            } else false;
-
-            if (!reply_in_core) return reply;
-        }
-
-        return null;
-    }
-
-    /// The cluster was unable to upgrade because one or more of its reformat clients were evicted.
-    /// This is not strictly related to the core -- an upgrade requires all (non-standby) replicas.
-    pub fn core_reformat_evicted(simulator: *const Simulator) bool {
-        assert(simulator.core.count() > 0);
-
-        const eviction_reasons = simulator.cluster.client_eviction_reasons;
-        const eviction_reasons_reformats =
-            eviction_reasons[simulator.cluster.options.client_count..];
-        assert(eviction_reasons_reformats.len == simulator.cluster.options.reformats_max);
-
-        for (eviction_reasons_reformats) |reason_or_null| {
-            if (reason_or_null) |reason| {
-                log.err("reformat evicted with {s}", .{@tagName(reason)});
-                assert(reason == .no_session or reason == .session_too_low);
-                return true;
-            }
-        }
-        return false;
-    }
-
-    fn core_release_max(simulator: *const Simulator) vsr.Release {
-        assert(simulator.core.count() > 0);
-
-        var release_max: vsr.Release = vsr.Release.zero;
-        for (simulator.cluster.replicas) |*replica| {
-            if (simulator.core.is_set(replica.replica)) {
-                release_max = release_max.max(replica.release);
-                if (replica.upgrade_release) |release| {
-                    release_max = release_max.max(release);
-                }
-            }
-        }
-        assert(release_max.value > 0);
-        return release_max;
-    }
-
-    fn on_cluster_reply(
-        cluster: *Cluster,
-        reply_client: ?usize,
-        prepare: *const Message.Prepare,
-        reply: *const Message.Reply,
-    ) void {
-        assert((reply_client == null) == (prepare.header.client == 0));
-
-        const simulator: *Simulator = @ptrCast(@alignCast(cluster.context.?));
-
-        if (reply.header.op < simulator.reply_op_next) return;
-        if (simulator.reply_sequence.contains(reply)) return;
-
-        simulator.reply_sequence.insert(reply_client, prepare, reply);
-
-        while (!simulator.reply_sequence.empty()) {
-            const op = simulator.reply_op_next;
-            const prepare_header = simulator.cluster.state_checker.commits.items[op].header;
-            assert(prepare_header.op == op);
-
-            if (simulator.reply_sequence.peek(op)) |commit| {
-                defer simulator.reply_sequence.next();
-
-                simulator.reply_op_next += 1;
-
-                assert(commit.reply.references == 1);
-                assert(commit.reply.header.op == op);
-                assert(commit.reply.header.command == .reply);
-                assert(commit.reply.header.request == commit.prepare.header.request);
-                assert(commit.reply.header.operation == commit.prepare.header.operation);
-                assert(commit.prepare.references == 1);
-                assert(commit.prepare.header.checksum == prepare_header.checksum);
-                assert(commit.prepare.header.command == .prepare);
-
-                log.debug("consume_stalled_replies: op={} operation={} client={} request={}", .{
-                    commit.reply.header.op,
-                    commit.reply.header.operation,
-                    commit.prepare.header.client,
-                    commit.prepare.header.request,
-                });
-
-                if (prepare_header.operation == .pulse) {
-                    simulator.workload.on_pulse(
-                        prepare_header.operation.cast(StateMachine.Operation),
-                        prepare_header.timestamp,
-                    );
-                }
-
-                if (!commit.prepare.header.operation.vsr_reserved()) {
-                    simulator.workload.on_reply(
-                        commit.client_index.?,
-                        commit.reply.header.operation.cast(StateMachine.Operation),
-                        commit.reply.header.timestamp,
-                        commit.prepare.body_used(),
-                        commit.reply.body_used(),
-                    );
-                }
-            }
-        }
-    }
-
-    fn on_client_reply(
-        cluster: *Cluster,
-        reply_client: usize,
-        request: *const Message.Request,
-        reply: *const Message.Reply,
-    ) void {
-        _ = reply;
-
-        const simulator: *Simulator = @ptrCast(@alignCast(cluster.context.?));
-        assert(simulator.cluster.client_eviction_reasons[reply_client] == null);
-
-        if (!request.header.operation.vsr_reserved()) {
-            simulator.requests_replied += 1;
-        }
-    }
-
-    /// Maybe send a request from one of the cluster's clients.
-    fn tick_requests(simulator: *Simulator) void {
-        if (simulator.requests_idle) {
-            if (simulator.prng.chance(simulator.options.request_idle_off_probability)) {
-                simulator.requests_idle = false;
-            }
-        } else {
-            if (simulator.prng.chance(simulator.options.request_idle_on_probability)) {
-                simulator.requests_idle = true;
-            }
-        }
-
-        if (simulator.requests_idle) return;
-        if (simulator.requests_sent - simulator.cluster.client_eviction_requests_cancelled ==
-            simulator.options.requests_max) return;
-        if (!simulator.prng.chance(simulator.options.request_probability)) return;
-
-        const client_index = index: {
-            const client_count = simulator.options.cluster.client_count;
-            const client_index_base =
-                simulator.prng.int_inclusive(usize, client_count - 1);
-            for (0..client_count) |offset| {
-                const client_index = (client_index_base + offset) % client_count;
-                if (simulator.cluster.client_eviction_reasons[client_index] == null) {
-                    break :index client_index;
-                }
-            } else {
-                for (0..client_count) |index| {
-                    assert(simulator.cluster.client_eviction_reasons[index] != null);
-                    assert(simulator.cluster.client_eviction_reasons[index] == .no_session or
-                        simulator.cluster.client_eviction_reasons[index] == .session_too_low);
-                }
-                unimplemented("client replacement; all clients were evicted");
-            }
-        };
-
-        var client = &simulator.cluster.clients[client_index].?;
-
-        // Messages aren't added to the ReplySequence until a reply arrives.
-        // Before sending a new message, make sure there will definitely be room for it.
-        var reserved: usize = 0;
-        for (
-            simulator.cluster.clients,
-        ) |*client_maybe| {
-            if (client_maybe.*) |*c| {
-                // Count the number of clients that are still waiting for a `register` to complete,
-                // since they may start one at any time.
-                reserved += @intFromBool(c.session == 0);
-                // Count the number of non-register requests queued.
-                reserved += @intFromBool(c.request_inflight != null);
-            }
-        }
-        // +1 for the potential request — is there room in the sequencer's queue?
-        if (reserved + 1 > simulator.reply_sequence.free()) return;
-
-        // Make sure that the client is ready to send a new request.
-        if (client.request_inflight != null) return;
-        const request_message = client.get_message();
-        errdefer client.release_message(request_message);
-
-        const request_metadata = simulator.workload.build_request(
-            client_index,
-            request_message.buffer[@sizeOf(vsr.Header)..constants.message_size_max],
-        );
-        assert(request_metadata.size <= constants.message_body_size_max);
-
-        simulator.cluster.request(
-            client_index,
-            request_metadata.operation,
-            request_message,
-            request_metadata.size,
-        );
-        // Since we already checked the client's request queue for free space, `client.request()`
-        // should always queue the request.
-        assert(request_message == client.request_inflight.?.message.base());
-        assert(request_message.header.size == @sizeOf(vsr.Header) + request_metadata.size);
-        assert(request_message.header.into(.request).?.operation.cast(StateMachine.Operation) ==
-            request_metadata.operation);
-
-        simulator.requests_sent += 1;
-        assert(simulator.requests_sent - simulator.cluster.client_eviction_requests_cancelled <=
-            simulator.options.requests_max);
-    }
-
-    fn tick_upgrade(simulator: *Simulator) void {
-        for (simulator.cluster.replicas) |*replica| {
-            const upgrade =
-                simulator.replica_releases[replica.replica] < releases.len and
-                simulator.prng.chance(simulator.options.replica_release_advance_probability);
-            if (upgrade) simulator.replica_upgrade(replica.replica);
-        }
-    }
-
-    fn tick_crash(simulator: *Simulator) void {
-        for (simulator.cluster.replicas) |*replica| {
-            simulator.replica_crash_stability[replica.replica] -|= 1;
-            if (simulator.replica_crash_stability[replica.replica] > 0) continue;
-
-            switch (simulator.cluster.replica_health[replica.replica]) {
-                .up => |up| {
-                    if (!up.paused) simulator.tick_crash_up(replica);
-                },
-                .down => simulator.tick_crash_down(replica),
-                .reformatting => {},
-            }
-        }
-    }
-
-    fn tick_crash_up(simulator: *Simulator, replica: *Cluster.Replica) void {
-        const replica_storage = &simulator.cluster.storages[replica.replica];
-        const replica_writes = replica_storage.writes.count();
-
-        var crash_probability = simulator.options.replica_crash_probability;
-        if (replica_writes > 0) crash_probability.numerator *= 10;
-
-        const crash_random = simulator.prng.chance(crash_probability);
-
-        if (!crash_random) return;
-
-        log.debug("{}: crash replica", .{replica.replica});
-        simulator.cluster.replica_crash(replica.replica);
-
-        simulator.replica_crash_stability[replica.replica] =
-            simulator.options.replica_crash_stability;
-    }
-
-    fn tick_crash_down(simulator: *Simulator, replica: *Cluster.Replica) void {
-        // If we are in liveness mode, we need to make sure that all replicas
-        // (eventually) make it to the same release.
-        const restart_upgrade =
-            simulator.replica_releases[replica.replica] <
-            simulator.replica_releases_limit and
-            (simulator.core.is_set(replica.replica) or
-                simulator.prng.chance(simulator.options.replica_release_catchup_probability));
-        if (restart_upgrade) simulator.replica_upgrade(replica.replica);
-
-        const restart_random =
-            simulator.prng.chance(simulator.options.replica_restart_probability);
-
-        if (!restart_upgrade and !restart_random) return;
-
-        const recoverable_count_min =
-            vsr.quorums(simulator.options.cluster.replica_count).view_change;
-
-        var recoverable_count: usize = 0;
-        for (simulator.cluster.replicas, 0..) |*r, i| {
-            recoverable_count += @intFromBool(simulator.cluster.replica_health[i] == .up and
-                !simulator.replica_reformats.is_set(replica.replica) and
-                !r.standby() and
-                r.status != .recovering_head and
-                r.syncing == .idle);
-        }
-
-        // To improve VOPR utilization, try to prevent the replica from going into
-        // `.recovering_head` state if the replica is needed to form a quorum.
-        const fault = recoverable_count >= recoverable_count_min or replica.standby();
-        if (fault) {
-            const reformat_random =
-                !replica.standby() and
-                simulator.cluster.reformat_count < simulator.cluster.options.reformats_max and
-                simulator.prng.chance(simulator.options.replica_reformat_probability);
-            if (reformat_random) {
-                log.debug("{}: reformat replica", .{replica.replica});
-
-                simulator.replica_reformats.set(replica.replica);
-                simulator.cluster.replica_reformat(replica.replica) catch unreachable;
-                return;
-            }
-        }
-        simulator.replica_restart(replica.replica, fault);
-        maybe(!fault and replica.status == .recovering_head);
-    }
-
-    fn replica_restart(simulator: *Simulator, replica_index: u8, fault: bool) void {
-        assert(simulator.cluster.replica_health[replica_index] == .down);
-
-        const replica_storage = &simulator.cluster.storages[replica_index];
-        const replica: *const Cluster.Replica = &simulator.cluster.replicas[replica_index];
-
-        {
-            // If the entire Zone.wal_headers is corrupted, the replica becomes permanently
-            // unavailable (returns `WALInvalid` from `open`). In the simulator, there are only two
-            // WAL sectors, which could both get corrupted when a replica crashes while writing them
-            // simultaneously. Repair both sectors so that even if one of them becomes corrupted on
-            // startup, the replica still remains operational.
-            //
-            // In production `journal_iops_write_max < header_sector_count`, which makes is
-            // impossible to get torn writes for all journal header sectors at the same time.
-            const header_sector_offset =
-                @divExact(vsr.Zone.wal_headers.start(), constants.sector_size);
-            const header_sector_count =
-                @divExact(constants.journal_size_headers, constants.sector_size);
-            for (0..header_sector_count) |header_sector_index| {
-                replica_storage.faults.unset(header_sector_offset + header_sector_index);
-            }
-            // TODO Clear misdirects? Waiting for a seed to confirm.
-        }
-
-        var header_prepare_view_mismatch: bool = false;
-        if (!fault) {
-            // The journal writes redundant headers of faulty ops as zeroes to ensure
-            // that they remain faulty after a crash/recover. Since that fault cannot
-            // be disabled by `storage.faulty`, we must manually repair it here to
-            // ensure a cluster cannot become stuck in status=recovering_head.
-            // See recover_slots() for more detail.
-            const headers_offset = vsr.Zone.wal_headers.offset(0);
-            const headers_size = vsr.Zone.wal_headers.size().?;
-            const headers_bytes = replica_storage.memory[headers_offset..][0..headers_size];
-            for (
-                mem.bytesAsSlice(vsr.Header.Prepare, headers_bytes),
-                replica_storage.wal_prepares(),
-            ) |*wal_header, *wal_prepare| {
-                if (wal_header.checksum == 0) {
-                    wal_header.* = wal_prepare.header;
-                } else {
-                    if (wal_header.view != wal_prepare.header.view) {
-                        header_prepare_view_mismatch = true;
-                    }
-                }
-            }
-        }
-
-        log.debug("{}: restart replica (faults={} releases={})", .{
-            replica_index,
-            fault,
-            simulator.replica_releases[replica_index],
-        });
-
-        replica_storage.faulty = fault;
-        simulator.cluster.replica_restart(replica_index) catch unreachable;
-
-        if (replica.status == .recovering_head) {
-            // Even with faults disabled, a replica may wind up in status=recovering_head.
-            assert(fault or header_prepare_view_mismatch);
-        }
-
-        replica_storage.faulty = true;
-        simulator.replica_crash_stability[replica_index] =
-            simulator.options.replica_restart_stability;
-    }
-
-    fn replica_upgrade(simulator: *Simulator, replica_index: u8) void {
-        simulator.replica_releases[replica_index] =
-            @min(simulator.replica_releases[replica_index] + 1, releases.len);
-        simulator.replica_releases_limit =
-            @max(simulator.replica_releases[replica_index], simulator.replica_releases_limit);
-
-        const replica_releases = simulator.replica_release_list(replica_index);
-        simulator.cluster.replica_set_releases(replica_index, &replica_releases);
-    }
-
-    fn replica_release_list(simulator: *const Simulator, replica_index: u8) vsr.ReleaseList {
-        const replica_releases_count = simulator.replica_releases[replica_index];
-        var release_list: vsr.ReleaseList = .empty;
-        for (0..replica_releases_count) |i| {
-            release_list.push(releases[i].release);
-        }
-        release_list.verify();
-        return release_list;
-    }
-
-    // Randomly pause replicas. A paused replica doesn't tick and doesn't complete any asynchronous
-    // work. The goals of pausing are:
-    // - catch more interesting interleaving of events,
-    // - simulate real-world scenario of VM migration.
-    fn tick_pause(simulator: *Simulator) void {
-        for (
-            simulator.cluster.replicas,
-            simulator.replica_crash_stability,
-            0..,
-        ) |*replica, *stability, replica_index| {
-            stability.* -|= 1;
-            if (stability.* > 0) continue;
-
-            if (simulator.cluster.replica_health[replica.replica] != .up) continue;
-            const paused = simulator.cluster.replica_health[replica.replica].up.paused;
-            const pause = simulator.prng.chance(simulator.options.replica_pause_probability);
-            const unpause = simulator.prng.chance(simulator.options.replica_unpause_probability);
-
-            if (!paused and pause) {
-                simulator.cluster.replica_pause(@intCast(replica_index));
-                stability.* = simulator.options.replica_pause_stability;
-            } else if (paused and unpause) {
-                simulator.cluster.replica_unpause(@intCast(replica_index));
-                stability.* = simulator.options.replica_unpause_stability;
-            }
-        }
-    }
-};
-
-/// Print an error message and then exit with an exit code.
-fn fatal(failure: Failure, comptime fmt_string: []const u8, args: anytype) noreturn {
-    log.err(fmt_string, args);
-    std.process.exit(@intFromEnum(failure));
-}
-
-/// Signal that something is not yet fully implemented, and abort the process.
-///
-/// In VOPR, this will exit with status 0, to make it easy to find "real" failures by running
-/// the simulator in a loop.
-fn unimplemented(comptime message: []const u8) noreturn {
-    const full_message = "unimplemented: " ++ message;
-    log.info(full_message, .{});
-    log.info("not crashing in VOPR", .{});
-    std.process.exit(0);
-}
-
-/// Returns a random fully-connected subgraph which includes at least view change
-/// quorum of active replicas.
-fn random_core(prng: *stdx.PRNG, replica_count: u8, standby_count: u8) Core {
-    assert(replica_count > 0);
-    assert(replica_count <= constants.replicas_max);
-    assert(standby_count <= constants.standbys_max);
-
-    const quorum_view_change = vsr.quorums(replica_count).view_change;
-    const replica_core_count = prng.range_inclusive(u8, quorum_view_change, replica_count);
-    const standby_core_count = prng.range_inclusive(u8, 0, standby_count);
-
-    var core: Core = .{};
-
-    var combination = stdx.PRNG.Combination.init(.{
-        .total = replica_count,
-        .sample = replica_core_count,
-    });
-    for (0..replica_count) |replica| {
-        if (combination.take(prng)) core.set(replica);
-    }
-    assert(combination.done());
-
-    combination = stdx.PRNG.Combination.init(.{
-        .total = standby_count,
-        .sample = standby_core_count,
-    });
-    for (replica_count..replica_count + standby_count) |standby| {
-        if (combination.take(prng)) core.set(standby);
-    }
-    assert(combination.done());
-
-    assert(core.count() == replica_core_count + standby_core_count);
-
-    return core;
-}
-
-/// Returns a random fully-connected subgraph which includes all replicas and standbys.
-fn full_core(replica_count: u8, standby_count: u8) Core {
-    assert(replica_count > 0);
-    assert(replica_count <= constants.replicas_max);
-    assert(standby_count <= constants.standbys_max);
-
-    var core: Core = .{};
-
-    for (0..replica_count + standby_count) |replica| core.set(replica);
-
-    assert(core.count() == replica_count + standby_count);
-
-    return core;
-}
-
-var log_buffer: std.io.BufferedWriter(4096, std.fs.File.Writer) = .{
-    // This is initialized in main(), as std.io.getStdErr() is not comptime known on e.g. Windows.
-    .unbuffered_writer = undefined,
-};
-
-var log_performance_mode: bool = false;
-
-fn log_override(
-    comptime level: std.log.Level,
-    comptime scope: @TypeOf(.enum_literal),
-    comptime format: []const u8,
-    args: anytype,
-) void {
-    if ((scope == .vsr and level == .err) or scope == .gpa) {
-        // Always print a message for vsr.fatal.
-    } else {
-        if (vsr_vopr_options.log == .short) {
-            if (log_performance_mode) {
-                if (scope != .simulator) return;
-            } else {
-                if (scope != .simulator and scope != .cluster) return;
-            }
-        }
-    }
-
-    const prefix_default = "[" ++ @tagName(level) ++ "] " ++ "(" ++ @tagName(scope) ++ "): ";
-    const prefix = if (vsr_vopr_options.log == .short) "" else prefix_default;
-
-    // Print the message to stderr using a buffer to avoid many small write() syscalls when
-    // providing many format arguments. Silently ignore failure.
-    log_buffer.writer().print(prefix ++ format ++ "\n", args) catch {};
-
-    // Flush the buffer before returning to ensure, for example, that a log message
-    // immediately before a failing assertion is fully printed.
-    log_buffer.flush() catch {};
-}