tigerbeetle 0.0.40 → 0.17.8

data/ext/tb_client/tigerbeetle/src/ewah_benchmark.zig

@@ -1,128 +0,0 @@
-const std = @import("std");
-const assert = std.debug.assert;
-const ewah = @import("ewah.zig").ewah(usize);
-const stdx = @import("stdx");
-
-const log = std.log;
-
-const BitSetConfig = struct {
-    words: usize,
-    run_length_e: usize,
-    literals_length_e: usize,
-};
-
-// Bump these up if you want to use this as a real benchmark rather than as a test.
-const samples = 10;
-const repeats: usize = 1_000;
-
-// Explanation of fields:
-// - "n": Number of randomly generate bitsets to test.
-// - "words": The length of the decoded bitset, in u64s.
-// - "run_length_e": The expected length of a run, ignoring truncation due to reaching the end of
-//   the bitset.
-// - "literals_length_e": Expected length of a sequence of literals.
-const configs = [_]BitSetConfig{
-    // primarily runs
-    .{ .words = 640, .run_length_e = 10, .literals_length_e = 10 },
-    .{ .words = 640, .run_length_e = 100, .literals_length_e = 10 },
-    .{ .words = 640, .run_length_e = 200, .literals_length_e = 10 },
-    // primarily literals
-    .{ .words = 640, .run_length_e = 1, .literals_length_e = 100 },
-};
-
-var prng = stdx.PRNG.from_seed(42);
-
-test "benchmark: ewah" {
-    for (configs) |config| {
-        var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
-        defer arena.deinit();
-
-        const allocator = arena.allocator();
-        var i: usize = 0;
-        var bitsets: [samples][]usize = undefined;
-        var bitsets_encoded: [samples][]align(@alignOf(usize)) u8 = undefined;
-        var bitsets_decoded: [samples][]usize = undefined;
-        var bitset_lengths: [samples]usize = undefined;
-        while (i < samples) : (i += 1) {
-            bitsets[i] = try make_bitset(allocator, config);
-            bitsets_encoded[i] = try allocator.alignedAlloc(
-                u8,
-                @alignOf(usize),
-                ewah.encode_size_max(bitsets[i].len),
-            );
-            bitsets_decoded[i] = try allocator.alloc(usize, config.words);
-        }
-
-        // Benchmark encoding.
-        var encode_timer = try std.time.Timer.start();
-        i = 0;
-        while (i < samples) : (i += 1) {
-            var j: usize = 0;
-            var size: usize = undefined;
-            while (j < repeats) : (j += 1) {
-                size = ewah.encode_all(bitsets[i], bitsets_encoded[i]);
-            }
-            bitset_lengths[i] = size;
-        }
-        const encode_time = encode_timer.read() / samples / repeats;
-
-        var decode_timer = try std.time.Timer.start();
-        // Benchmark decoding.
-        i = 0;
-        while (i < samples) : (i += 1) {
-            const bitset_encoded = bitsets_encoded[i][0..bitset_lengths[i]];
-            var j: usize = 0;
-            while (j < repeats) : (j += 1) {
-                _ = ewah.decode_all(bitset_encoded, bitsets_decoded[i]);
-            }
-        }
-        const decode_time = decode_timer.read() / samples / repeats;
-
-        i = 0;
-        while (i < samples) : (i += 1) {
-            assert(std.mem.eql(usize, bitsets[i], bitsets_decoded[i]));
-        }
-
-        // Compute compression ratio.
-        var total_uncompressed: f64 = 0.0;
-        var total_compressed: f64 = 0.0;
-        i = 0;
-        while (i < samples) : (i += 1) {
-            total_uncompressed += @as(f64, @floatFromInt(bitsets[i].len * @sizeOf(usize)));
-            total_compressed += @as(f64, @floatFromInt(bitset_lengths[i]));
-        }
-
-        log.info(
-            \\Words={:_>3} E(Run)={:_>3} E(Literal)={:_>3} EncTime={:_>6}ns DecTime={:_>6}ns Ratio={d:_>6.2}
-        , .{
-            config.words,
-            config.run_length_e,
-            config.literals_length_e,
-            encode_time,
-            decode_time,
-            total_uncompressed / total_compressed,
-        });
-    }
-}
-
-fn make_bitset(allocator: std.mem.Allocator, config: BitSetConfig) ![]usize {
-    var words = try allocator.alloc(usize, config.words);
-    var w: usize = 0;
-    var literal: usize = 1;
-    while (w < words.len) : (w += 1) {
-        const run_length = prng.int_inclusive(usize, (2 * config.run_length_e) - 1);
-        const literals_length = prng.int_inclusive(usize, (2 * config.literals_length_e) - 1);
-        const run_bit = prng.boolean();
-
-        const run_end = @min(w + run_length, words.len);
-        while (w < run_end) : (w += 1) {
-            words[w] = if (run_bit) std.math.maxInt(usize) else 0;
-        }
-        const literals_end = @min(w + literals_length, words.len);
-        while (w < literals_end) : (w += 1) {
-            words[w] = literal;
-            literal += 1;
-        }
-    }
-    return words;
-}

data/ext/tb_client/tigerbeetle/src/ewah_fuzz.zig

@@ -1,171 +0,0 @@
-//! Fuzz EWAH encode/decode cycle.
-const std = @import("std");
-const assert = std.debug.assert;
-const log = std.log.scoped(.fuzz_ewah);
-
-const stdx = @import("stdx");
-
-const ewah = @import("./ewah.zig");
-const fuzz = @import("./testing/fuzz.zig");
-
-const MiB = stdx.MiB;
-
-pub fn main(gpa: std.mem.Allocator, args: fuzz.FuzzArgs) !void {
-    inline for (.{ u8, u16, u32, u64, usize }) |Word| {
-        var prng = stdx.PRNG.from_seed(args.seed);
-
-        const decoded_size_max = @divExact(1 * MiB, @sizeOf(Word));
-        const decoded_size = prng.range_inclusive(usize, 1, decoded_size_max);
-        const decoded = try gpa.alloc(Word, decoded_size);
-        defer gpa.free(decoded);
-
-        const decoded_bits_total = decoded_size * @bitSizeOf(Word);
-        const decoded_bits = prng.int_inclusive(usize, decoded_bits_total);
-        generate_bits(&prng, std.mem.sliceAsBytes(decoded[0..decoded_size]), decoded_bits);
-
-        var context = try ContextType(Word).init(gpa, decoded.len);
-        defer context.deinit(gpa);
-
-        const encode_chunk_words_count = prng.range_inclusive(usize, 1, decoded_size);
-        const decode_chunk_words_count = prng.range_inclusive(usize, 1, decoded_size);
-
-        const encoded_size = try context.test_encode_decode(decoded, .{
-            .encode_chunk_words_count = encode_chunk_words_count,
-            .decode_chunk_words_count = decode_chunk_words_count,
-        });
-
-        log.info("word={} decoded={} encoded={} compression_ratio={d:.2} set={d:.2} " ++
-            "encode_chunk={} decode_chunk={}", .{
-            Word,
-            decoded_size,
-            encoded_size,
-            @as(f64, @floatFromInt(decoded_size)) / @as(f64, @floatFromInt(encoded_size)),
-            @as(f64, @floatFromInt(decoded_bits)) / @as(f64, @floatFromInt(decoded_bits_total)),
-            encode_chunk_words_count,
-            decode_chunk_words_count,
-        });
-    }
-}
-
-pub fn fuzz_encode_decode(
-    comptime Word: type,
-    gpa: std.mem.Allocator,
-    decoded: []const Word,
-    options: ContextType(Word).TestOptions,
-) !void {
-    var context = try ContextType(Word).init(gpa, decoded.len);
-    defer context.deinit(gpa);
-
-    _ = try context.test_encode_decode(decoded, options);
-}
-
-/// Modify `data` such that it has exactly `bits_set_total` randomly-chosen bits set,
-/// with the remaining bits unset.
-fn generate_bits(prng: *stdx.PRNG, data: []u8, bits_set_total: usize) void {
-    const bits_total = data.len * @bitSizeOf(u8);
-    assert(bits_set_total <= bits_total);
-
-    // Start off full or empty to save some work.
-    const init_empty = bits_set_total < @divExact(bits_total, 2);
-    @memset(data, if (init_empty) @as(u8, 0) else std.math.maxInt(u8));
-
-    var bits_set = if (init_empty) 0 else bits_total;
-    while (bits_set != bits_set_total) {
-        const bit = prng.int_inclusive(usize, bits_total - 1);
-        const word = @divFloor(bit, @bitSizeOf(u8));
-        const mask = @as(u8, 1) << @as(std.math.Log2Int(u8), @intCast(bit % @bitSizeOf(u8)));
-
-        if (init_empty) {
-            if (data[word] & mask != 0) continue;
-            data[word] |= mask;
-            bits_set += 1;
-        } else {
-            if (data[word] & mask == 0) continue;
-            data[word] &= ~mask;
-            bits_set -= 1;
-        }
-    }
-}
-
-pub fn ContextType(comptime Word: type) type {
-    return struct {
-        const Context = @This();
-        const Codec = ewah.ewah(Word);
-
-        decoded_actual: []Word,
-        encoded_actual: []align(@alignOf(Word)) u8,
-
-        fn init(gpa: std.mem.Allocator, size_max: usize) !Context {
-            const decoded_actual = try gpa.alloc(Word, size_max);
-            errdefer gpa.free(decoded_actual);
-
-            const encoded_actual = try gpa.alignedAlloc(
-                u8,
-                @alignOf(Word),
-                Codec.encode_size_max(size_max),
-            );
-            errdefer gpa.free(encoded_actual);
-
-            return Context{
-                .decoded_actual = decoded_actual,
-                .encoded_actual = encoded_actual,
-            };
-        }
-
-        fn deinit(context: *Context, gpa: std.mem.Allocator) void {
-            gpa.free(context.decoded_actual);
-            gpa.free(context.encoded_actual);
-        }
-
-        pub const TestOptions = struct {
-            encode_chunk_words_count: usize,
-            decode_chunk_words_count: usize,
-        };
-
-        fn test_encode_decode(
-            context: Context,
-            decoded_expect: []const Word,
-            options: TestOptions,
-        ) !usize {
-            assert(decoded_expect.len > 0);
-
-            var encoder = Codec.encode_chunks(decoded_expect);
-            var encoded_size: usize = 0;
-            while (!encoder.done()) {
-                const chunk_words_count = @min(
-                    @divExact(context.encoded_actual.len - encoded_size, @sizeOf(Word)),
-                    options.encode_chunk_words_count,
-                );
-
-                const chunk =
-                    context.encoded_actual[encoded_size..][0 .. chunk_words_count * @sizeOf(Word)];
-
-                encoded_size += encoder.encode_chunk(@alignCast(chunk));
-            }
-
-            var decoder = Codec.decode_chunks(context.decoded_actual[0..], encoded_size);
-            var decoded_actual_size: usize = 0;
-            var decoder_input_offset: usize = 0;
-            while (decoder_input_offset < encoded_size) {
-                const chunk_size = @min(
-                    encoded_size - decoder_input_offset,
-                    options.decode_chunk_words_count * @sizeOf(Word),
-                );
-
-                const chunk = context.encoded_actual[decoder_input_offset..][0..chunk_size];
-
-                decoded_actual_size += decoder.decode_chunk(@alignCast(chunk));
-                decoder_input_offset += chunk_size;
-            }
-            assert(decoder.done());
-
-            try std.testing.expectEqual(decoded_expect.len, decoded_actual_size);
-            try std.testing.expectEqualSlices(
-                Word,
-                decoded_expect,
-                context.decoded_actual[0..decoded_actual_size],
-            );
-            return encoded_size;
-        }
-    };
-}

data/ext/tb_client/tigerbeetle/src/fuzz_tests.zig

@@ -1,179 +0,0 @@
-const std = @import("std");
-const assert = std.debug.assert;
-
-const stdx = @import("stdx");
-const constants = @import("./constants.zig");
-const fuzz = @import("./testing/fuzz.zig");
-
-const log = std.log.scoped(.fuzz);
-
-// This enables `constants.verify` for this file.
-pub const vsr_options = .{
-    .config_verify = true,
-    .git_commit = @import("vsr_options").git_commit,
-    .release = @import("vsr_options").release,
-    .release_client_min = @import("vsr_options").release_client_min,
-};
-
-// NB: this changes values in `constants.zig`!
-pub const tigerbeetle_config = @import("config.zig").configs.test_min;
-comptime {
-    assert(constants.storage_size_limit_max == tigerbeetle_config.process.storage_size_limit_max);
-}
-
-pub const std_options: std.Options = .{
-    .log_level = .info,
-    .log_scope_levels = &[_]std.log.ScopeLevel{
-        .{ .scope = .message_bus, .level = .err },
-        .{ .scope = .storage, .level = .err },
-        .{ .scope = .superblock_quorums, .level = .err },
-        .{ .scope = .state_machine, .level = .err },
-    },
-};
-
-const Fuzzers = .{
-    .ewah = @import("./ewah_fuzz.zig"),
-    .lsm_scan = @import("./lsm/scan_fuzz.zig"),
-    .lsm_cache_map = @import("./lsm/cache_map_fuzz.zig"),
-    .lsm_forest = @import("./lsm/forest_fuzz.zig"),
-    .lsm_manifest_log = @import("./lsm/manifest_log_fuzz.zig"),
-    .lsm_manifest_level = @import("./lsm/manifest_level_fuzz.zig"),
-    .lsm_segmented_array = @import("./lsm/segmented_array_fuzz.zig"),
-    .lsm_tree = @import("./lsm/tree_fuzz.zig"),
-    .message_bus = @import("./message_bus_fuzz.zig"),
-    .storage = @import("./storage_fuzz.zig"),
-    .vsr_free_set = @import("./vsr/free_set_fuzz.zig"),
-    .vsr_superblock = @import("./vsr/superblock_fuzz.zig"),
-    .vsr_superblock_quorums = @import("./vsr/superblock_quorums_fuzz.zig"),
-    .vsr_multi_batch = @import("./vsr/multi_batch_fuzz.zig"),
-    .signal = @import("./clients/c/tb_client/signal_fuzz.zig"),
-    .state_machine = @import("./state_machine_fuzz.zig"),
-    // A fuzzer that intentionally fails, to test fuzzing infrastructure itself
-    .canary = {},
-    // Quickly run all fuzzers as a smoke test
-    .smoke = {},
-};
-
-const FuzzersEnum = std.meta.FieldEnum(@TypeOf(Fuzzers));
-
-const CLIArgs = struct {
-    events_max: ?usize = null,
-
-    @"--": void,
-    fuzzer: FuzzersEnum,
-    seed: ?u64 = null,
-};
-
-pub fn main() !void {
-    comptime assert(constants.verify);
-
-    fuzz.limit_ram();
-
-    var gpa_allocator: std.heap.GeneralPurposeAllocator(.{}) = .{};
-    // Disable "hint" argument for mmap call, which was observed to cause stack overflow.
-    // See https://ziggit.dev/t/stack-probe-puzzle/10291/3 for the full story.
-    gpa_allocator.backing_allocator = .{
-        .ptr = std.heap.page_allocator.ptr,
-        .vtable = &comptime .{
-            .alloc = struct {
-                fn alloc(
-                    ctx: *anyopaque,
-                    len: usize,
-                    ptr_align: std.mem.Alignment,
-                    ret_addr: usize,
-                ) ?[*]u8 {
-                    @atomicStore(
-                        @TypeOf(std.heap.next_mmap_addr_hint),
-                        &std.heap.next_mmap_addr_hint,
-                        null,
-                        .monotonic,
-                    );
-                    return std.heap.page_allocator.vtable.alloc(ctx, len, ptr_align, ret_addr);
-                }
-            }.alloc,
-            .remap = std.heap.page_allocator.vtable.remap,
-            .resize = std.heap.page_allocator.vtable.resize,
-            .free = std.heap.page_allocator.vtable.free,
-        },
-    };
-    const gpa = gpa_allocator.allocator();
-
-    var args = try std.process.argsWithAllocator(gpa);
-    defer args.deinit();
-
-    const cli_args = stdx.flags(&args, CLIArgs);
-
-    switch (cli_args.fuzzer) {
-        .smoke => {
-            assert(cli_args.seed == null);
-            assert(cli_args.events_max == null);
-            try main_smoke(gpa);
-        },
-        else => try main_single(gpa, cli_args),
-    }
-}
-
-fn main_smoke(gpa: std.mem.Allocator) !void {
-    var timer_all = try std.time.Timer.start();
-    inline for (comptime std.enums.values(FuzzersEnum)) |fuzzer| {
-        const events_max = switch (fuzzer) {
-            .smoke => continue,
-            .canary => continue,
-
-            .lsm_cache_map => 20_000,
-            .lsm_forest => 10_000,
-            .lsm_manifest_log => 2_000,
-            .lsm_scan => 100,
-            .lsm_tree => 400,
-            .state_machine => 10_000,
-            .message_bus => 10,
-            .storage => 1_000,
-            .vsr_free_set => 10_000,
-            .vsr_multi_batch => 128,
-            .vsr_superblock => 3,
-
-            inline .ewah,
-            .lsm_segmented_array,
-            .lsm_manifest_level,
-            .vsr_superblock_quorums,
-            .signal,
-            => null,
-        };
-
-        var timer_single = try std.time.Timer.start();
-        try @field(Fuzzers, @tagName(fuzzer)).main(gpa, .{
-            .seed = 123,
-            .events_max = events_max,
-        });
-        const fuzz_duration = timer_single.lap();
-        if (fuzz_duration > 10 * std.time.ns_per_s) {
-            log.err("fuzzer too slow for the smoke mode: " ++ @tagName(fuzzer) ++ " {}", .{
-                std.fmt.fmtDuration(fuzz_duration),
-            });
-        }
-    }
-
-    log.info("done in {}", .{std.fmt.fmtDuration(timer_all.lap())});
-}
-
-fn main_single(gpa: std.mem.Allocator, cli_args: CLIArgs) !void {
-    assert(cli_args.fuzzer != .smoke);
-
-    const seed = cli_args.seed orelse std.crypto.random.int(u64);
-    log.info("Fuzz seed = {}", .{seed});
-
-    var timer = try std.time.Timer.start();
-    switch (cli_args.fuzzer) {
-        .smoke => unreachable,
-        .canary => {
-            if (seed % 100 == 0) {
-                std.process.exit(1);
-            }
-        },
-        inline else => |fuzzer| try @field(Fuzzers, @tagName(fuzzer)).main(gpa, .{
-            .seed = seed,
-            .events_max = cli_args.events_max,
-        }),
-    }
-    log.info("done in {}", .{std.fmt.fmtDuration(timer.lap())});
-}