tigerbeetle 0.0.40 → 0.17.8

data/ext/tb_client/tigerbeetle/src/testing/exhaustigen.zig

@@ -1,128 +0,0 @@
-//! An utility for exhaustive generation of arbitrary data
-//! <https://matklad.github.io/2021/11/07/generate-all-the-things.html>
-test "generate all permutations" {
-    var g: Gen = .{};
-    var permutation_count: u32 = 0;
-
-    // This loop imperatively generates all permutations of "abcd":
-    while (!g.done()) {
-        var pool_buffer: [4]u8 = "abcd".*;
-        var permutation: [4]u8 = undefined;
-
-        for (0..permutation.len) |i| {
-            const pool = pool_buffer[0 .. pool_buffer.len - i];
-
-            // Pick a "random" number from the pool, append it to the permutation,
-            // and swap-remove it from the pool.
-            const pool_index = g.index(pool);
-            permutation[i] = pool[pool_index];
-            pool[pool_index] = pool[pool.len - 1];
-        }
-        // Here, `permutation` enumerates all permutations of `abcd`:
-        // std.debug.print("permutation: {s}\n", .{permutation});
-        permutation_count += 1;
-    }
-
-    // Verify that we indeed generated n! permutations.
-    var factorial: usize = 1;
-    for (1..5) |n| factorial *= n;
-
-    assert(permutation_count == factorial);
-}
-
-const std = @import("std");
-const assert = std.debug.assert;
-
-// The implementation is tricky, refer to the post for details.
-//
-// On each iteration of `while (!g.done())` loop, Gen generates a sequence of numbers.
-// Internally, it remembers this sequence  together with bounds the user requested:
-//
-// value:  3 1 4 4
-// bound:  5 4 4 4
-//
-// To advance to the next iteration, Gen finds the smallest sequence of values which is larger than
-// the current one, but still satisfies all the bounds. "Smallest" means that Gen tries to increment
-// the rightmost number.
-//
-// In the above example, the last two "4"s already match the bound, so we can't increment them.
-// However, we can increment the second number, "1", to get 3 2 4 4. This isn’t the smallest
-// sequence though, 3 2 0 0 is be smaller. So, after incrementing the rightmost number possible,
-// we zero the rest.
-const Gen = @This();
-
-started: bool = false,
-v: [32]struct { value: u32, bound: u32 } = undefined,
-p: usize = 0,
-p_max: usize = 0,
-
-pub fn done(g: *@This()) bool {
-    if (!g.started) {
-        g.started = true;
-        return false;
-    }
-    var i = g.p_max;
-    while (i > 0) {
-        i -= 1;
-        if (g.v[i].value < g.v[i].bound) {
-            g.v[i].value += 1;
-            g.p_max = i + 1;
-            g.p = 0;
-            return false;
-        }
-    }
-    return true;
-}
-
-fn gen(g: *Gen, bound: u32) u32 {
-    assert(g.p < g.v.len);
-    if (g.p == g.p_max) {
-        g.v[g.p] = .{ .value = 0, .bound = 0 };
-        g.p_max += 1;
-    }
-    g.p += 1;
-    g.v[g.p - 1].bound = bound;
-    return g.v[g.p - 1].value;
-}
-
-pub fn int_inclusive(g: *Gen, Int: type, bound: Int) Int {
-    return @intCast(g.gen(@intCast(bound)));
-}
-
-pub fn range_inclusive(g: *Gen, Int: type, min: Int, max: Int) Int {
-    comptime assert(@typeInfo(Int).int.signedness == .unsigned);
-    assert(min <= max);
-    return min + g.int_inclusive(Int, max - min);
-}
-
-pub fn shuffle(g: *Gen, T: type, slice: []T) void {
-    for (0..slice.len) |i| {
-        const j = g.int_inclusive(u64, i);
-        std.mem.swap(T, &slice[i], &slice[j]);
-    }
-}
-
-test shuffle {
-    var n_factorial: u32 = 1;
-    inline for (0..5) |n| {
-        var g: Gen = .{};
-        var count: u32 = 0;
-        while (!g.done()) {
-            var array: [n]u8 = @splat(0);
-            g.shuffle(u8, &array);
-            count += 1;
-        }
-        assert(count == n_factorial);
-        n_factorial *= (n + 1);
-    }
-}
-
-pub fn index(g: *Gen, slice: anytype) usize {
-    assert(slice.len > 0);
-    return g.int_inclusive(usize, slice.len - 1);
-}
-
-pub fn enum_value(g: *Gen, Enum: type) Enum {
-    const values = std.enums.values(Enum);
-    return values[g.index(values)];
-}

data/ext/tb_client/tigerbeetle/src/testing/fixtures.zig

@@ -1,181 +0,0 @@
-//! Convenient constructs for TigerBeetle components, for fuzzing and testing.
-//!
-//! Consider the Grid. In the actual database, there is only single call to Grid.init.
-//! However, Grid is needed for most of our tests and fuzzers. If the init call is repeated
-//! in every fuzzer, changing Storage creation flow becomes hard. To solve this, all fuzzers create
-//! Grid through this file, such that we have one production and one test call to Grid.init.
-//!
-//! Design:
-//!
-//! - All functions take struct options as a last argument, even if it starts out as empty.
-//!   All call-sites pass at least .{}, which makes adding new options cheap.
-//! - Most options should have defaults. This is intentional deviation from TigerStyle, as, for
-//!   tests, we gain a useful property: all options that are set are meaningful for a particular
-//!   test.
-//! - All dependent fixtures are passed in positionally because they can't be defaulted and their
-//!   types are unique.
-//! - It could be convenient to export types themselves, in addition to constructors, but we avoid
-//!   introducing two ways to import something.
-const std = @import("std");
-const vsr = @import("../vsr.zig");
-const constants = @import("../constants.zig");
-const assert = std.debug.assert;
-
-const Time = @import("../time.zig").Time;
-const OffsetType = @import("./time.zig").OffsetType;
-const Tracer = @import("../trace.zig").Tracer;
-const Storage = @import("./storage.zig").Storage;
-const SuperBlock = vsr.SuperBlockType(Storage);
-const Grid = vsr.GridType(Storage);
-
-const TimeSim = @import("./time.zig").TimeSim;
-
-pub const cluster: u128 = 0;
-pub const replica: u8 = 0;
-pub const replica_count: u8 = 6;
-
-pub fn init_time(options: struct {
-    resolution: u64 = constants.tick_ms * std.time.ns_per_ms,
-    offset_type: OffsetType = .linear,
-    offset_coefficient_A: i64 = 0,
-    offset_coefficient_B: i64 = 0,
-    offset_coefficient_C: u32 = 0,
-}) TimeSim {
-    const result: TimeSim = .{
-        .resolution = options.resolution,
-        .offset_type = options.offset_type,
-        .offset_coefficient_A = options.offset_coefficient_A,
-        .offset_coefficient_B = options.offset_coefficient_B,
-        .offset_coefficient_C = options.offset_coefficient_C,
-    };
-    return result;
-}
-
-pub fn init_tracer(gpa: std.mem.Allocator, init: Time, options: struct {
-    writer: ?std.io.AnyWriter = null,
-    process_id: Tracer.ProcessID = .{ .replica = .{ .cluster = cluster, .replica = replica } },
-}) !Tracer {
-    return Tracer.init(gpa, init, options.process_id, .{ .writer = options.writer });
-}
-
-pub fn init_storage(gpa: std.mem.Allocator, options: Storage.Options) !Storage {
-    return try Storage.init(gpa, options);
-}
-
-pub fn storage_format(
-    gpa: std.mem.Allocator,
-    storage: *Storage,
-    options: struct {
-        cluster: u128 = cluster,
-        replica: u8 = replica,
-        replica_count: u8 = replica_count,
-        release: vsr.Release = vsr.Release.minimum,
-    },
-) !void {
-    assert(storage.reads.count() == 0);
-    assert(storage.writes.count() == 0);
-
-    var superblock = try init_superblock(gpa, storage, .{});
-    defer superblock.deinit(gpa);
-
-    const Context = struct {
-        superblock_context: SuperBlock.Context = undefined,
-        done: bool = false,
-        fn callback(superblock_context: *SuperBlock.Context) void {
-            const self: *@This() = @fieldParentPtr("superblock_context", superblock_context);
-            assert(!self.done);
-            self.done = true;
-        }
-    };
-    var context: Context = .{};
-
-    superblock.format(Context.callback, &context.superblock_context, .{
-        .cluster = options.cluster,
-        .replica = options.replica,
-        .replica_count = options.replica_count,
-        .release = options.release,
-        .view = null,
-    });
-    for (0..10_000) |_| {
-        if (context.done) break;
-        storage.run();
-    } else @panic("superblock format loop stuck");
-    assert(storage.reads.count() == 0);
-    assert(storage.writes.count() == 0);
-}
-
-pub fn init_superblock(gpa: std.mem.Allocator, storage: *Storage, options: struct {
-    storage_size_limit: ?u64 = null,
-}) !SuperBlock {
-    return try SuperBlock.init(gpa, storage, .{
-        .storage_size_limit = options.storage_size_limit orelse storage.size,
-    });
-}
-
-pub fn init_grid(gpa: std.mem.Allocator, trace: *Tracer, superblock: *SuperBlock, options: struct {
-    missing_blocks_max: u64 = 0,
-    missing_tables_max: u64 = 0,
-    blocks_released_prior_checkpoint_durability_max: u64 = 0,
-}) !Grid {
-    return try Grid.init(gpa, .{
-        .superblock = superblock,
-        .trace = trace,
-        .missing_blocks_max = options.missing_blocks_max,
-        .missing_tables_max = options.missing_tables_max,
-        .blocks_released_prior_checkpoint_durability_max = //
-        options.blocks_released_prior_checkpoint_durability_max,
-    });
-}
-
-pub fn open_superblock(superblock: *SuperBlock) void {
-    const storage: *Storage = superblock.storage;
-    assert(storage.reads.count() == 0);
-    assert(storage.writes.count() == 0);
-    defer assert(storage.reads.count() == 0);
-    defer assert(storage.writes.count() == 0);
-
-    const Context = struct {
-        superblock_context: SuperBlock.Context = undefined,
-        pending: bool = false,
-
-        fn callback(superblock_context: *SuperBlock.Context) void {
-            const self: *@This() = @fieldParentPtr("superblock_context", superblock_context);
-            assert(self.pending);
-            self.pending = false;
-        }
-    };
-    var context: Context = .{};
-
-    context.pending = true;
-    superblock.open(Context.callback, &context.superblock_context);
-    for (0..10_000) |_| {
-        storage.run();
-        if (!context.pending) break;
-    } else @panic("open superblock stuck");
-}
-
-pub fn open_grid(grid: *Grid) void {
-    const storage: *Storage = grid.superblock.storage;
-    assert(storage.reads.count() == 0);
-    assert(storage.writes.count() == 0);
-    defer assert(storage.reads.count() == 0);
-    defer assert(storage.writes.count() == 0);
-
-    const Context = struct {
-        pending: bool = false,
-        // NB: This lacks in elegance and robustness, but is good enough for testing.
-        var global: @This() = .{};
-        fn callback(_: *Grid) void {
-            assert(@This().global.pending);
-            @This().global.pending = false;
-        }
-    };
-
-    assert(!Context.global.pending);
-    Context.global.pending = true;
-    grid.open(Context.callback);
-    for (0..10_000) |_| {
-        storage.run();
-        if (!Context.global.pending) break;
-    } else @panic("open grid stuck");
-}

data/ext/tb_client/tigerbeetle/src/testing/fuzz.zig

@@ -1,144 +0,0 @@
-//! Utils functions for writing fuzzers.
-
-const builtin = @import("builtin");
-const std = @import("std");
-const stdx = @import("stdx");
-const assert = std.debug.assert;
-const PRNG = stdx.PRNG;
-const Duration = stdx.Duration;
-
-const GiB = stdx.GiB;
-
-const log = std.log.scoped(.fuzz);
-
-/// Returns an integer of type `T` with an exponential distribution of rate `avg`.
-/// Note: If you specify a very high rate then `std.math.maxInt(T)` may be over-represented.
-pub fn random_int_exponential(prng: *stdx.PRNG, comptime T: type, avg: T) T {
-    comptime {
-        const info = @typeInfo(T);
-        assert(info == .int);
-        assert(info.int.signedness == .unsigned);
-    }
-    // Note: we use floats and rely on std implementation. Ideally, we should do neither, but I
-    // wasn't able to find a quick way to generate geometrically distributed integers using only
-    // integer arithmetic.
-    const random = std.Random.init(prng, stdx.PRNG.fill);
-    const exp = random.floatExp(f64) * @as(f64, @floatFromInt(avg));
-    return std.math.lossyCast(T, exp);
-}
-
-/// Return a distribution for use with `random_enum`.
-///
-/// This is swarm testing: some variants are disabled completely,
-/// and the rest have wildly different probabilities.
-pub fn random_enum_weights(
-    prng: *stdx.PRNG,
-    comptime Enum: type,
-) stdx.PRNG.EnumWeightsType(Enum) {
-    const fields = comptime std.meta.fieldNames(Enum);
-
-    var combination = stdx.PRNG.Combination.init(.{
-        .total = fields.len,
-        .sample = prng.range_inclusive(u32, 1, fields.len),
-    });
-    defer assert(combination.done());
-
-    var weights: PRNG.EnumWeightsType(Enum) = undefined;
-    inline for (fields) |field| {
-        @field(weights, field) = if (combination.take(prng))
-            prng.range_inclusive(u64, 1, 100)
-        else
-            0;
-    }
-
-    return weights;
-}
-
-/// We have two opposing desires for prng ids:
-/// 1. We want to cause many collisions.
-/// 2. We want to generate enough ids that various caches can't hold them all.
-///
-/// So, flip a coin and pick an an ID either from a small, or from a large set.
-pub fn random_id(prng: *stdx.PRNG, comptime Int: type, options: struct {
-    average_hot: Int,
-    average_cold: Int,
-}) Int {
-    assert(options.average_hot < options.average_cold);
-    const average: Int = if (prng.boolean()) options.average_hot else options.average_cold;
-    return random_int_exponential(prng, Int, average);
-}
-
-pub fn range_inclusive_ms(prng: *stdx.PRNG, min: anytype, max: anytype) Duration {
-    const min_ns = switch (@TypeOf(min)) {
-        comptime_int, u64 => min * std.time.ns_per_ms,
-        Duration => min.ns,
-        else => comptime unreachable,
-    };
-    const max_ns = switch (@TypeOf(max)) {
-        comptime_int, u64 => max * std.time.ns_per_ms,
-        Duration => max.ns,
-        else => comptime unreachable,
-    };
-    return .{ .ns = prng.range_inclusive(u64, min_ns, max_ns) };
-}
-
-pub const FuzzArgs = struct {
-    seed: u64,
-    events_max: ?usize,
-};
-
-pub fn parse_seed(bytes: []const u8) u64 {
-    if (bytes.len == 40) {
-        // Normally, a seed is specified as a base-10 integer. However, as a special case, we allow
-        // using a Git hash (a hex string 40 character long). This is used by our CI, which passes
-        // current commit hash as a seed --- that way, we run simulator on CI, we run it with
-        // different, "random" seeds, but the failures remain reproducible just from the commit
-        // hash!
-        const commit_hash = std.fmt.parseUnsigned(u160, bytes, 16) catch |err| switch (err) {
-            error.Overflow => unreachable,
-            error.InvalidCharacter => @panic("commit hash seed contains an invalid character"),
-        };
-        return @truncate(commit_hash);
-    }
-
-    return std.fmt.parseUnsigned(u64, bytes, 10) catch |err| switch (err) {
-        error.Overflow => @panic("seed exceeds a 64-bit unsigned integer"),
-        error.InvalidCharacter => @panic("seed contains an invalid character"),
-    };
-}
-
-// Like `std.meta.DeclEnum`, but allows excluding specific things. Feed the result into
-// random_enum_weights for swarm testing public API of a data structure.
-pub fn DeclEnumExcludingType(T: type, exclude: []const std.meta.DeclEnum(T)) type {
-    const base = @typeInfo(std.meta.DeclEnum(T)).@"enum";
-    assert(exclude.len > 0); // Use plain std.meta.DeclEnum.
-    assert(exclude.len < base.fields.len);
-    var fields_filtered: [base.fields.len - exclude.len]std.builtin.Type.EnumField = undefined;
-    var i: usize = 0;
-    next_field: for (base.fields) |field| {
-        for (exclude) |excluded| {
-            if (std.mem.eql(u8, field.name, @tagName(excluded))) continue :next_field;
-        }
-        fields_filtered[i] = field;
-        i += 1;
-    }
-    assert(i == fields_filtered.len);
-
-    return @Type(.{ .@"enum" = .{
-        .tag_type = base.tag_type,
-        .fields = &fields_filtered,
-        .decls = &.{},
-        .is_exhaustive = true,
-    } });
-}
-
-pub fn limit_ram() void {
-    if (builtin.target.os.tag != .linux) return;
-
-    std.posix.setrlimit(.AS, .{
-        .cur = 20 * GiB,
-        .max = 20 * GiB,
-    }) catch |err| {
-        log.warn("failed to setrlimit address space: {}", .{err});
-    };
-}

data/ext/tb_client/tigerbeetle/src/testing/id.zig

@@ -1,97 +0,0 @@
-const std = @import("std");
-const stdx = @import("stdx");
-
-/// Permute indices (or other encoded data) into ids to:
-///
-/// * test different patterns of ids (e.g. random, ascending, descending), and
-/// * allow the original index to recovered from the id, enabling less stateful testing.
-///
-pub const IdPermutation = union(enum) {
-    /// Ascending indices become ascending ids.
-    identity: void,
-
-    /// Ascending indices become descending ids.
-    inversion: void,
-
-    /// Ascending indices alternate between ascending/descending (e.g. 1,100,3,98,…).
-    zigzag: void,
-
-    /// Ascending indices become pseudo-UUIDs.
-    ///
-    /// Sandwich the index "data" between random bits — this randomizes the id's prefix and suffix,
-    /// but the index is easily recovered:
-    ///
-    /// * id_bits[_0.._32] = random
-    /// * id_bits[32.._96] = data
-    /// * id_bits[96..128] = random
-    random: u64,
-
-    pub fn encode(self: *const IdPermutation, data: usize) u128 {
-        return switch (self.*) {
-            .identity => data,
-            .inversion => std.math.maxInt(u128) - @as(u128, data),
-            .zigzag => {
-                if (data % 2 == 0) {
-                    return data;
-                } else {
-                    // -1 to stay odd.
-                    return std.math.maxInt(u128) - @as(u128, data) -% 1;
-                }
-            },
-            .random => |seed| {
-                var prng = stdx.PRNG.from_seed(seed +% data);
-                const random_mask = ~@as(u128, std.math.maxInt(u64) << 32);
-                const random_bits = random_mask & prng.int(u128);
-                return @as(u128, data) << 32 | random_bits;
-            },
-        };
-    }
-
-    pub fn decode(self: *const IdPermutation, id: u128) usize {
-        return switch (self.*) {
-            .identity => @intCast(id),
-            .inversion => @intCast(std.math.maxInt(u128) - id),
-            .zigzag => {
-                if (id % 2 == 0) {
-                    return @intCast(id);
-                } else {
-                    // -1 to stay odd.
-                    return @intCast(std.math.maxInt(u128) - id -% 1);
-                }
-            },
-            .random => @truncate(id >> 32),
-        };
-    }
-
-    pub fn generate(prng: *stdx.PRNG) IdPermutation {
-        return switch (prng.enum_uniform(std.meta.Tag(IdPermutation))) {
-            .identity => .{ .identity = {} },
-            .inversion => .{ .inversion = {} },
-            .zigzag => .{ .zigzag = {} },
-            .random => .{ .random = prng.int(u64) },
-        };
-    }
-};
-
-test "IdPermutation" {
-    var prng = stdx.PRNG.from_seed_testing();
-
-    for ([_]IdPermutation{
-        .{ .identity = {} },
-        .{ .inversion = {} },
-        .{ .zigzag = {} },
-        .{ .random = prng.int(u64) },
-    }) |permutation| {
-        var i: usize = 0;
-        while (i < 20) : (i += 1) {
-            const r = prng.int(usize);
-            try test_id_permutation(permutation, r);
-            try test_id_permutation(permutation, i);
-            try test_id_permutation(permutation, std.math.maxInt(usize) - i);
-        }
-    }
-}
-
-fn test_id_permutation(permutation: IdPermutation, value: usize) !void {
-    try std.testing.expectEqual(value, permutation.decode(permutation.encode(value)));
-}