tigerbeetle 0.0.34 → 0.0.37

data/ext/tb_client/tigerbeetle/src/ewah_fuzz.zig

@@ -0,0 +1,171 @@
+//! Fuzz EWAH encode/decode cycle.
+const std = @import("std");
+const assert = std.debug.assert;
+const log = std.log.scoped(.fuzz_ewah);
+
+const stdx = @import("stdx");
+
+const ewah = @import("./ewah.zig");
+const fuzz = @import("./testing/fuzz.zig");
+
+const MiB = stdx.MiB;
+
+pub fn main(gpa: std.mem.Allocator, args: fuzz.FuzzArgs) !void {
+    inline for (.{ u8, u16, u32, u64, usize }) |Word| {
+        var prng = stdx.PRNG.from_seed(args.seed);
+
+        const decoded_size_max = @divExact(1 * MiB, @sizeOf(Word));
+        const decoded_size = prng.range_inclusive(usize, 1, decoded_size_max);
+        const decoded = try gpa.alloc(Word, decoded_size);
+        defer gpa.free(decoded);
+
+        const decoded_bits_total = decoded_size * @bitSizeOf(Word);
+        const decoded_bits = prng.int_inclusive(usize, decoded_bits_total);
+        generate_bits(&prng, std.mem.sliceAsBytes(decoded[0..decoded_size]), decoded_bits);
+
+        var context = try ContextType(Word).init(gpa, decoded.len);
+        defer context.deinit(gpa);
+
+        const encode_chunk_words_count = prng.range_inclusive(usize, 1, decoded_size);
+        const decode_chunk_words_count = prng.range_inclusive(usize, 1, decoded_size);
+
+        const encoded_size = try context.test_encode_decode(decoded, .{
+            .encode_chunk_words_count = encode_chunk_words_count,
+            .decode_chunk_words_count = decode_chunk_words_count,
+        });
+
+        log.info("word={} decoded={} encoded={} compression_ratio={d:.2} set={d:.2} " ++
+            "encode_chunk={} decode_chunk={}", .{
+            Word,
+            decoded_size,
+            encoded_size,
+            @as(f64, @floatFromInt(decoded_size)) / @as(f64, @floatFromInt(encoded_size)),
+            @as(f64, @floatFromInt(decoded_bits)) / @as(f64, @floatFromInt(decoded_bits_total)),
+            encode_chunk_words_count,
+            decode_chunk_words_count,
+        });
+    }
+}
+
+pub fn fuzz_encode_decode(
+    comptime Word: type,
+    gpa: std.mem.Allocator,
+    decoded: []const Word,
+    options: ContextType(Word).TestOptions,
+) !void {
+    var context = try ContextType(Word).init(gpa, decoded.len);
+    defer context.deinit(gpa);
+
+    _ = try context.test_encode_decode(decoded, options);
+}
+
+/// Modify `data` such that it has exactly `bits_set_total` randomly-chosen bits set,
+/// with the remaining bits unset.
+fn generate_bits(prng: *stdx.PRNG, data: []u8, bits_set_total: usize) void {
+    const bits_total = data.len * @bitSizeOf(u8);
+    assert(bits_set_total <= bits_total);
+
+    // Start off full or empty to save some work.
+    const init_empty = bits_set_total < @divExact(bits_total, 2);
+    @memset(data, if (init_empty) @as(u8, 0) else std.math.maxInt(u8));
+
+    var bits_set = if (init_empty) 0 else bits_total;
+    while (bits_set != bits_set_total) {
+        const bit = prng.int_inclusive(usize, bits_total - 1);
+        const word = @divFloor(bit, @bitSizeOf(u8));
+        const mask = @as(u8, 1) << @as(std.math.Log2Int(u8), @intCast(bit % @bitSizeOf(u8)));
+
+        if (init_empty) {
+            if (data[word] & mask != 0) continue;
+            data[word] |= mask;
+            bits_set += 1;
+        } else {
+            if (data[word] & mask == 0) continue;
+            data[word] &= ~mask;
+            bits_set -= 1;
+        }
+    }
+}
+
+pub fn ContextType(comptime Word: type) type {
+    return struct {
+        const Context = @This();
+        const Codec = ewah.ewah(Word);
+
+        decoded_actual: []Word,
+        encoded_actual: []align(@alignOf(Word)) u8,
+
+        fn init(gpa: std.mem.Allocator, size_max: usize) !Context {
+            const decoded_actual = try gpa.alloc(Word, size_max);
+            errdefer gpa.free(decoded_actual);
+
+            const encoded_actual = try gpa.alignedAlloc(
+                u8,
+                @alignOf(Word),
+                Codec.encode_size_max(size_max),
+            );
+            errdefer gpa.free(encoded_actual);
+
+            return Context{
+                .decoded_actual = decoded_actual,
+                .encoded_actual = encoded_actual,
+            };
+        }
+
+        fn deinit(context: *Context, gpa: std.mem.Allocator) void {
+            gpa.free(context.decoded_actual);
+            gpa.free(context.encoded_actual);
+        }
+
+        pub const TestOptions = struct {
+            encode_chunk_words_count: usize,
+            decode_chunk_words_count: usize,
+        };
+
+        fn test_encode_decode(
+            context: Context,
+            decoded_expect: []const Word,
+            options: TestOptions,
+        ) !usize {
+            assert(decoded_expect.len > 0);
+
+            var encoder = Codec.encode_chunks(decoded_expect);
+            var encoded_size: usize = 0;
+            while (!encoder.done()) {
+                const chunk_words_count = @min(
+                    @divExact(context.encoded_actual.len - encoded_size, @sizeOf(Word)),
+                    options.encode_chunk_words_count,
+                );
+
+                const chunk =
+                    context.encoded_actual[encoded_size..][0 .. chunk_words_count * @sizeOf(Word)];
+
+                encoded_size += encoder.encode_chunk(@alignCast(chunk));
+            }
+
+            var decoder = Codec.decode_chunks(context.decoded_actual[0..], encoded_size);
+            var decoded_actual_size: usize = 0;
+            var decoder_input_offset: usize = 0;
+            while (decoder_input_offset < encoded_size) {
+                const chunk_size = @min(
+                    encoded_size - decoder_input_offset,
+                    options.decode_chunk_words_count * @sizeOf(Word),
+                );
+
+                const chunk = context.encoded_actual[decoder_input_offset..][0..chunk_size];
+
+                decoded_actual_size += decoder.decode_chunk(@alignCast(chunk));
+                decoder_input_offset += chunk_size;
+            }
+            assert(decoder.done());
+
+            try std.testing.expectEqual(decoded_expect.len, decoded_actual_size);
+            try std.testing.expectEqualSlices(
+                Word,
+                decoded_expect,
+                context.decoded_actual[0..decoded_actual_size],
+            );
+            return encoded_size;
+        }
+    };
+}

data/ext/tb_client/tigerbeetle/src/fuzz_tests.zig

@@ -0,0 +1,179 @@
+const std = @import("std");
+const assert = std.debug.assert;
+
+const stdx = @import("stdx");
+const constants = @import("./constants.zig");
+const fuzz = @import("./testing/fuzz.zig");
+
+const log = std.log.scoped(.fuzz);
+
+// This enables `constants.verify` for this file.
+pub const vsr_options = .{
+    .config_verify = true,
+    .git_commit = @import("vsr_options").git_commit,
+    .release = @import("vsr_options").release,
+    .release_client_min = @import("vsr_options").release_client_min,
+};
+
+// NB: this changes values in `constants.zig`!
+pub const tigerbeetle_config = @import("config.zig").configs.test_min;
+comptime {
+    assert(constants.storage_size_limit_max == tigerbeetle_config.process.storage_size_limit_max);
+}
+
+pub const std_options: std.Options = .{
+    .log_level = .info,
+    .log_scope_levels = &[_]std.log.ScopeLevel{
+        .{ .scope = .message_bus, .level = .err },
+        .{ .scope = .storage, .level = .err },
+        .{ .scope = .superblock_quorums, .level = .err },
+        .{ .scope = .state_machine, .level = .err },
+    },
+};
+
+const Fuzzers = .{
+    .ewah = @import("./ewah_fuzz.zig"),
+    .lsm_scan = @import("./lsm/scan_fuzz.zig"),
+    .lsm_cache_map = @import("./lsm/cache_map_fuzz.zig"),
+    .lsm_forest = @import("./lsm/forest_fuzz.zig"),
+    .lsm_manifest_log = @import("./lsm/manifest_log_fuzz.zig"),
+    .lsm_manifest_level = @import("./lsm/manifest_level_fuzz.zig"),
+    .lsm_segmented_array = @import("./lsm/segmented_array_fuzz.zig"),
+    .lsm_tree = @import("./lsm/tree_fuzz.zig"),
+    .message_bus = @import("./message_bus_fuzz.zig"),
+    .storage = @import("./storage_fuzz.zig"),
+    .vsr_free_set = @import("./vsr/free_set_fuzz.zig"),
+    .vsr_superblock = @import("./vsr/superblock_fuzz.zig"),
+    .vsr_superblock_quorums = @import("./vsr/superblock_quorums_fuzz.zig"),
+    .vsr_multi_batch = @import("./vsr/multi_batch_fuzz.zig"),
+    .signal = @import("./clients/c/tb_client/signal_fuzz.zig"),
+    .state_machine = @import("./state_machine_fuzz.zig"),
+    // A fuzzer that intentionally fails, to test fuzzing infrastructure itself
+    .canary = {},
+    // Quickly run all fuzzers as a smoke test
+    .smoke = {},
+};
+
+const FuzzersEnum = std.meta.FieldEnum(@TypeOf(Fuzzers));
+
+const CLIArgs = struct {
+    events_max: ?usize = null,
+
+    @"--": void,
+    fuzzer: FuzzersEnum,
+    seed: ?u64 = null,
+};
+
+pub fn main() !void {
+    comptime assert(constants.verify);
+
+    fuzz.limit_ram();
+
+    var gpa_allocator: std.heap.GeneralPurposeAllocator(.{}) = .{};
+    // Disable "hint" argument for mmap call, which was observed to cause stack overflow.
+    // See https://ziggit.dev/t/stack-probe-puzzle/10291/3 for the full story.
+    gpa_allocator.backing_allocator = .{
+        .ptr = std.heap.page_allocator.ptr,
+        .vtable = &comptime .{
+            .alloc = struct {
+                fn alloc(
+                    ctx: *anyopaque,
+                    len: usize,
+                    ptr_align: std.mem.Alignment,
+                    ret_addr: usize,
+                ) ?[*]u8 {
+                    @atomicStore(
+                        @TypeOf(std.heap.next_mmap_addr_hint),
+                        &std.heap.next_mmap_addr_hint,
+                        null,
+                        .monotonic,
+                    );
+                    return std.heap.page_allocator.vtable.alloc(ctx, len, ptr_align, ret_addr);
+                }
+            }.alloc,
+            .remap = std.heap.page_allocator.vtable.remap,
+            .resize = std.heap.page_allocator.vtable.resize,
+            .free = std.heap.page_allocator.vtable.free,
+        },
+    };
+    const gpa = gpa_allocator.allocator();
+
+    var args = try std.process.argsWithAllocator(gpa);
+    defer args.deinit();
+
+    const cli_args = stdx.flags(&args, CLIArgs);
+
+    switch (cli_args.fuzzer) {
+        .smoke => {
+            assert(cli_args.seed == null);
+            assert(cli_args.events_max == null);
+            try main_smoke(gpa);
+        },
+        else => try main_single(gpa, cli_args),
+    }
+}
+
+fn main_smoke(gpa: std.mem.Allocator) !void {
+    var timer_all = try std.time.Timer.start();
+    inline for (comptime std.enums.values(FuzzersEnum)) |fuzzer| {
+        const events_max = switch (fuzzer) {
+            .smoke => continue,
+            .canary => continue,
+
+            .lsm_cache_map => 20_000,
+            .lsm_forest => 10_000,
+            .lsm_manifest_log => 2_000,
+            .lsm_scan => 100,
+            .lsm_tree => 400,
+            .state_machine => 10_000,
+            .message_bus => 10,
+            .storage => 1_000,
+            .vsr_free_set => 10_000,
+            .vsr_multi_batch => 128,
+            .vsr_superblock => 3,
+
+            inline .ewah,
+            .lsm_segmented_array,
+            .lsm_manifest_level,
+            .vsr_superblock_quorums,
+            .signal,
+            => null,
+        };
+
+        var timer_single = try std.time.Timer.start();
+        try @field(Fuzzers, @tagName(fuzzer)).main(gpa, .{
+            .seed = 123,
+            .events_max = events_max,
+        });
+        const fuzz_duration = timer_single.lap();
+        if (fuzz_duration > 10 * std.time.ns_per_s) {
+            log.err("fuzzer too slow for the smoke mode: " ++ @tagName(fuzzer) ++ " {}", .{
+                std.fmt.fmtDuration(fuzz_duration),
+            });
+        }
+    }
+
+    log.info("done in {}", .{std.fmt.fmtDuration(timer_all.lap())});
+}
+
+fn main_single(gpa: std.mem.Allocator, cli_args: CLIArgs) !void {
+    assert(cli_args.fuzzer != .smoke);
+
+    const seed = cli_args.seed orelse std.crypto.random.int(u64);
+    log.info("Fuzz seed = {}", .{seed});
+
+    var timer = try std.time.Timer.start();
+    switch (cli_args.fuzzer) {
+        .smoke => unreachable,
+        .canary => {
+            if (seed % 100 == 0) {
+                std.process.exit(1);
+            }
+        },
+        inline else => |fuzzer| try @field(Fuzzers, @tagName(fuzzer)).main(gpa, .{
+            .seed = seed,
+            .events_max = cli_args.events_max,
+        }),
+    }
+    log.info("done in {}", .{std.fmt.fmtDuration(timer.lap())});
+}