tigerbeetle 0.0.40 → 0.17.8

data/ext/tb_client/tigerbeetle/src/testing/bench.zig

@@ -1,146 +0,0 @@
-//! Micro benchmarking harness.
-//!
-//! Goals:
-//! - relative (comparative) benchmarking,
-//! - manual checks when refactoring/optimizing/upgrading compiler,
-//! - no benchmark bitrot.
-//!
-//! Non-goals:
-//! - absolute benchmarking,
-//! - continuous benchmarking,
-//! - automatic regression detection.
-//!
-//! If you run
-//!     $ ./zig/zig build test
-//! the benchmarks are run in "test" mode which uses small inputs, finishes quickly, and doesn't
-//! print anything to stdout.
-//!
-//! If you run
-//!     $ ./zig/zig build -Drelease test -- "benchmark: binary search"
-//! the benchmark is run for real, with a large input, longer runtime, and results on stderr.
-//! The `benchmark` in the test name is the secret code to unlock benchmarking code.
-
-test "benchmark: API tutorial" { // `benchmark:` in the name is important!
-    var bench: Bench = .init();
-    defer bench.deinit();
-
-    // Parameters are named, and have two default values.
-    // The small value is used in tests, to prevent bitrot.
-    // The large value is the canonical when running benchmark "for real".
-    // You can pass custom values via env variables:
-    //    $ a=92 ./zig/zig build test -- "benchmark: tutorial"
-    const a = bench.parameter("a", 1, 1_000);
-    const b = bench.parameter("b", 2, 2_000);
-
-    bench.start(); // Built-in timer.
-    const c = a + b;
-    const elapsed = bench.stop();
-
-    // Always print a "hash" of the run:
-    // - to prevent compiler from optimizing the code away,
-    // - to prevent YOU from "optimizing" the code by changing semantics.
-    bench.report("hash: {}", .{c});
-    // Print the time, and any other metrics you find important.
-    bench.report("elapsed: {}", .{elapsed});
-
-    // NB: print as little as possible, because humans read slowly.
-    // It's the job of benchmark author to optimize for conciseness.
-
-    // You can compile individual benchmark  via
-    //   ./zig/zig build test:unit:build -- "benchmark: binary search"
-    // and use the resulting binary with perf/hyperfine/poop.
-}
-
-const std = @import("std");
-const assert = std.debug.assert;
-const stdx = @import("stdx");
-const Duration = stdx.Duration;
-const Instant = stdx.Instant;
-const TimeOS = @import("../time.zig").TimeOS;
-
-const seed_benchmark: u64 = 42;
-
-const mode: enum { smoke, benchmark } =
-    // See build.zig for how this is ultimately determined.
-    if (@import("test_options").benchmark) .benchmark else .smoke;
-
-seed: u64,
-time: TimeOS = .{},
-instant_start: ?Instant = null,
-
-const Bench = @This();
-
-pub fn init() Bench {
-    return .{
-        // Benchmarks require a fixed seed for reproducibility; smoke mode uses a random seed.
-        .seed = if (mode == .benchmark) seed_benchmark else std.testing.random_seed,
-    };
-}
-
-pub fn deinit(bench: *Bench) void {
-    assert(bench.instant_start == null);
-    bench.* = undefined;
-}
-
-pub fn parameter(
-    b: *const Bench,
-    comptime name: []const u8,
-    value_smoke: u64,
-    value_benchmark: u64,
-) u64 {
-    assert(value_smoke < value_benchmark);
-    const value = parameter_fallible(name, value_smoke, value_benchmark) catch |err| switch (err) {
-        error.InvalidCharacter, error.Overflow => @panic("invalid benchmark parameter value"),
-    };
-    b.report("{s}={}", .{ name, value });
-    return value;
-}
-
-fn parameter_fallible(
-    comptime name: []const u8,
-    value_smoke: u64,
-    value_benchmark: u64,
-) std.fmt.ParseIntError!u64 {
-    assert(value_smoke < value_benchmark);
-    return switch (mode) {
-        .smoke => value_smoke,
-        .benchmark => std.process.parseEnvVarInt(name, u64, 10) catch |err| switch (err) {
-            error.EnvironmentVariableNotFound => return value_benchmark,
-            else => |e| return e,
-        },
-    };
-}
-
-pub fn start(bench: *Bench) void {
-    assert(bench.instant_start == null);
-    defer assert(bench.instant_start != null);
-
-    bench.instant_start = bench.time.time().monotonic();
-}
-
-pub fn stop(bench: *Bench) Duration {
-    assert(bench.instant_start != null);
-    defer assert(bench.instant_start == null);
-
-    const instant_stop = bench.time.time().monotonic();
-    const elapsed = instant_stop.duration_since(bench.instant_start.?);
-    bench.instant_start = null;
-    return elapsed;
-}
-
-// Sort the durations and return the third-fastest sample (discarding the two fastest outliers)
-// to get a more stable estimate, assuming benchmark timings are roughly log-normal.
-// E.g. see https://lemire.me/blog/2018/01/16/microbenchmarking-calls-for-idealized-conditions/
-pub fn estimate(bench: *const Bench, durations: []Duration) Duration {
-    assert(durations.len >= 8); // Ensure that we have enough samples to get a meaningful result.
-    _ = bench;
-    std.sort.block(stdx.Duration, durations, {}, stdx.Duration.sort.asc);
-    return durations[2];
-}
-
-pub fn report(_: *const Bench, comptime fmt: []const u8, args: anytype) void {
-    switch (mode) {
-        .smoke => {},
-        .benchmark => std.debug.print(fmt ++ "\n", args),
-    }
-}

data/ext/tb_client/tigerbeetle/src/testing/cluster/grid_checker.zig

@@ -1,53 +0,0 @@
-const std = @import("std");
-const assert = std.debug.assert;
-const vsr = @import("../../vsr.zig");
-
-pub const GridChecker = struct {
-    const Blocks = std.AutoHashMap(struct {
-        checkpoint_id: u128,
-        block_address: u64,
-        checkpoint_durable: bool,
-    }, u128);
-
-    blocks: Blocks,
-
-    pub fn init(allocator: std.mem.Allocator) GridChecker {
-        return .{ .blocks = Blocks.init(allocator) };
-    }
-
-    pub fn deinit(checker: *GridChecker) void {
-        checker.blocks.deinit();
-    }
-
-    pub fn assert_coherent(
-        checker: *GridChecker,
-        checkpoint: *vsr.CheckpointState,
-        checkpoint_durable: bool,
-        block_address: u64,
-        block_checksum: u128,
-    ) void {
-        const result = checker.blocks.getOrPut(.{
-            .checkpoint_id = vsr.checksum(std.mem.asBytes(checkpoint)),
-            .block_address = block_address,
-            .checkpoint_durable = checkpoint_durable,
-        }) catch unreachable;
-
-        if (result.found_existing) {
-            assert(result.value_ptr.* == block_checksum);
-        } else {
-            result.value_ptr.* = block_checksum;
-        }
-
-        // Assert that the same version of the block must exist while the current checkpoint is
-        // not durable and while the previous checkpoint is durable.
-        if (!checkpoint_durable) {
-            if (checker.blocks.get(.{
-                .checkpoint_id = checkpoint.parent_checkpoint_id,
-                .block_address = block_address,
-                .checkpoint_durable = true,
-            })) |checksum| {
-                assert(checksum == block_checksum);
-            }
-        }
-    }
-};

data/ext/tb_client/tigerbeetle/src/testing/cluster/journal_checker.zig

@@ -1,61 +0,0 @@
-//! Verify Journal/WAL properties.
-const std = @import("std");
-const assert = std.debug.assert;
-const log = std.log.scoped(.journal_checker);
-
-const constants = @import("../../constants.zig");
-const stdx = @import("stdx");
-const vsr = @import("../../vsr.zig");
-const TestStorage = @import("../storage.zig").Storage;
-
-pub fn JournalCheckerType(comptime Replica: type) type {
-    return struct {
-        pub fn check(replica: *const Replica) void {
-            const replica_index = replica.replica;
-            const replica_storage = replica.superblock.storage;
-            comptime assert(@TypeOf(replica_storage) == *TestStorage);
-
-            if (replica.journal.writes.executing() == 0) {
-                // Sanity-check: Where the journal is clean and not being written to:
-                // - Redundant headers exactly match their corresponding prepares.
-                // - The WAL's content matches `journal.headers`.
-                // - There are no zeroed entries (representing faulty journal entries) in the
-                //   redundant WAL headers.
-                var wal_header_errors: u32 = 0;
-                for (
-                    replica_storage.wal_headers(),
-                    replica_storage.wal_prepares(),
-                    replica.journal.headers,
-                    0..,
-                ) |*wal_header, *wal_prepare, *journal_header, slot| {
-                    if (!replica.journal.dirty.bit(.{ .index = slot })) {
-                        if (journal_header.operation == .reserved) {
-                            // Ignore reserved headers -- when Journal.remove_entries_from()
-                            // truncates the log, it cleans the in-memory journal without writing to
-                            // the WAL.
-                        } else {
-                            if (wal_header.checksum == 0) {
-                                log.err("{}: check: slot={} checksum=0", .{ replica_index, slot });
-                                wal_header_errors += 1;
-                            } else {
-                                assert(wal_header.checksum == wal_prepare.header.checksum);
-                                assert(wal_header.checksum == journal_header.checksum);
-                            }
-                        }
-                    }
-                }
-                assert(wal_header_errors == 0);
-
-                // Verify that prepares' trailing sector padding is zeroed.
-                for (0..constants.journal_slot_count) |slot| {
-                    const prepare =
-                        replica_storage.area_memory(.{ .wal_prepares = .{ .slot = slot } });
-                    const header =
-                        std.mem.bytesAsValue(vsr.Header, prepare[0..@sizeOf(vsr.Header)]);
-                    const prepare_padding = prepare[header.size..vsr.sector_ceil(header.size)];
-                    assert(stdx.zeroed(prepare_padding));
-                }
-            }
-        }
-    };
-}

data/ext/tb_client/tigerbeetle/src/testing/cluster/manifest_checker.zig

@@ -1,76 +0,0 @@
-//! Verify that the ManifestLevels tables are constructed consistently across replicas and after
-//! recovering from a restart.
-const std = @import("std");
-const assert = std.debug.assert;
-
-const vsr = @import("../../vsr.zig");
-const constants = @import("../../constants.zig");
-
-pub fn ManifestCheckerType(comptime Forest: type) type {
-    return struct {
-        const ManifestChecker = @This();
-
-        /// Maps checkpoint op to the cumulative checksum of all trees/levels checksum.
-        const Checkpoints = std.AutoHashMap(u64, u128);
-
-        checkpoints: Checkpoints,
-
-        pub fn init(allocator: std.mem.Allocator) ManifestChecker {
-            return .{ .checkpoints = Checkpoints.init(allocator) };
-        }
-
-        pub fn deinit(checker: *ManifestChecker) void {
-            checker.checkpoints.deinit();
-        }
-
-        pub fn forest_open(checker: *ManifestChecker, forest: *const Forest) void {
-            checker.check(forest);
-        }
-
-        pub fn forest_checkpoint(checker: *ManifestChecker, forest: *const Forest) void {
-            checker.check(forest);
-        }
-
-        fn check(checker: *ManifestChecker, forest: *const Forest) void {
-            assert(forest.grid.superblock.opened);
-            assert(forest.manifest_log.opened);
-
-            const checkpoint_op = forest.grid.superblock.working.vsr_state.checkpoint.header.op;
-            const checksum_stored = checker.checkpoints.getOrPut(checkpoint_op) catch @panic("oom");
-            const checksum_current = manifest_levels_checksum(forest);
-
-            // On open, we will usually have already have a checksum to compare against from a prior
-            // checkpoint. But not always: it is possible that we are recovering from a checkpoint
-            // that wrote e.g. 3/4 superblock copies and then crashed.
-            if (checksum_stored.found_existing) {
-                assert(checksum_stored.value_ptr.* == checksum_current);
-            } else {
-                checksum_stored.value_ptr.* = checksum_current;
-            }
-        }
-
-        fn manifest_levels_checksum(forest: *const Forest) u128 {
-            var checksum_stream = vsr.ChecksumStream.init();
-            for (0..constants.lsm_levels) |level| {
-                checksum_stream.add(std.mem.asBytes(&level));
-
-                inline for (Forest.tree_id_range.min..Forest.tree_id_range.max + 1) |tree_id_u16| {
-                    const tree_id: Forest.TreeID = @enumFromInt(tree_id_u16);
-                    const tree_level = forest.tree_for_id_const(tree_id).manifest.levels[level];
-                    var tree_tables = tree_level.tables.iterator_from_index(0, .ascending);
-
-                    checksum_stream.add(std.mem.asBytes(&tree_id));
-                    checksum_stream.add(std.mem.asBytes(&tree_level.table_count_visible));
-                    while (tree_tables.next()) |tree_table| {
-                        checksum_stream.add(std.mem.asBytes(&tree_table.encode(.{
-                            .tree_id = tree_id_u16,
-                            .event = .insert, // (Placeholder event).
-                            .level = @intCast(level),
-                        })));
-                    }
-                }
-            }
-            return checksum_stream.checksum();
-        }
-    };
-}

data/ext/tb_client/tigerbeetle/src/testing/cluster/message_bus.zig

@@ -1,110 +0,0 @@
-const std = @import("std");
-const assert = std.debug.assert;
-
-const MessagePool = @import("../../message_pool.zig").MessagePool;
-const Message = MessagePool.Message;
-const MessageBuffer = @import("../../message_buffer.zig").MessageBuffer;
-const vsr = @import("../../vsr.zig");
-const ProcessType = vsr.ProcessType;
-
-const Network = @import("network.zig").Network;
-
-pub const Process = union(ProcessType) {
-    replica: u8,
-    client: u128,
-};
-
-pub const MessageBus = struct {
-    network: *Network,
-    pool: *MessagePool,
-
-    process: Process,
-
-    buffer: ?MessageBuffer,
-    suspended: bool = false,
-    resume_scheduled: bool = false,
-    /// The callback to be called when a message is received.
-    on_messages_callback: *const fn (message_bus: *MessageBus, buffer: *MessageBuffer) void,
-
-    pub const Options = struct {
-        network: *Network,
-    };
-
-    pub fn init(
-        _: std.mem.Allocator,
-        process: Process,
-        message_pool: *MessagePool,
-        on_messages_callback: *const fn (message_bus: *MessageBus, buffer: *MessageBuffer) void,
-        options: Options,
-    ) !MessageBus {
-        return MessageBus{
-            .network = options.network,
-            .pool = message_pool,
-            .process = process,
-            .buffer = MessageBuffer.init(message_pool),
-            .on_messages_callback = on_messages_callback,
-        };
-    }
-
-    pub fn deinit(bus: *MessageBus, _: std.mem.Allocator) void {
-        bus.buffer.?.deinit(bus.pool);
-        bus.buffer = null;
-        bus.resume_scheduled = false;
-        // NB: Network keeps a reference to a message bus even when a replica is de-initialized,
-        // so we don't assign bus.* to undefined here.
-    }
-
-    pub fn trace_gauge(_: *MessageBus) void {}
-
-    pub fn listen(_: *MessageBus) !void {}
-
-    pub fn tick(_: *MessageBus) void {}
-
-    pub fn tick_client(bus: *MessageBus) void {
-        bus.tick();
-    }
-
-    pub fn get_message(
-        bus: *MessageBus,
-        comptime command: ?vsr.Command,
-    ) MessagePool.GetMessageType(command) {
-        return bus.pool.get_message(command);
-    }
-
-    /// `@TypeOf(message)` is one of:
-    /// - `*Message`
-    /// - `MessageType(command)` for any `command`.
-    pub fn unref(bus: *MessageBus, message: anytype) void {
-        bus.pool.unref(message);
-    }
-
-    pub fn resume_needed(bus: *MessageBus) bool {
-        return bus.suspended;
-    }
-
-    pub fn resume_receive(bus: *MessageBus) void {
-        bus.suspended = false;
-        bus.resume_scheduled = true;
-    }
-
-    pub fn send_message_to_replica(bus: *MessageBus, replica: u8, message: *Message) void {
-        // Messages sent by a process to itself should never be passed to the message bus
-        if (bus.process == .replica) assert(replica != bus.process.replica);
-
-        bus.network.send_message(message, .{
-            .source = bus.process,
-            .target = .{ .replica = replica },
-        });
-    }
-
-    /// Try to send the message to the client with the given id.
-    /// If the client is not currently connected, the message is silently dropped.
-    pub fn send_message_to_client(bus: *MessageBus, client_id: u128, message: *Message) void {
-        assert(bus.process == .replica);
-
-        bus.network.send_message(message, .{
-            .source = bus.process,
-            .target = .{ .client = client_id },
-        });
-    }
-};