tigerbeetle 0.0.36 → 0.0.38

data/ext/tb_client/tigerbeetle/src/testing/io.zig

@@ -0,0 +1,317 @@
+const std = @import("std");
+const posix = std.posix;
+const mem = std.mem;
+const assert = std.debug.assert;
+
+const stdx = @import("stdx");
+const constants = @import("../constants.zig");
+const QueueType = @import("../queue.zig").QueueType;
+const buffer_limit = @import("../io.zig").buffer_limit;
+const Ratio = stdx.PRNG.Ratio;
+
+/// A very simple mock IO implementation that only implements what is needed to test Storage.
+pub const IO = struct {
+    pub const fd_t = u32;
+
+    pub const File = struct {
+        buffer: []u8,
+
+        /// Each bit of the fault map represents a sector that will fault consistently.
+        fault_map: ?[]const u8 = null,
+
+        closed: bool = false,
+
+        // Maintained for appending to the end of the file via `write_blocking`.
+        offset: u32 = 0,
+    };
+
+    /// Options for fault injection during fuzz testing.
+    pub const Options = struct {
+        /// Seed for the storage PRNG.
+        seed: u64 = 0,
+
+        /// Chance out of 100 that a read larger than a logical sector
+        /// will return an error.InputOutput.
+        larger_than_logical_sector_read_fault_probability: Ratio = Ratio.zero(),
+    };
+
+    const Queue = QueueType(Completion);
+
+    files: []File,
+
+    options: Options,
+    prng: stdx.PRNG,
+
+    completed: Queue = Queue.init(.{ .name = "io_completed" }),
+
+    pub fn init(files: []File, options: Options) !IO {
+        return .{
+            .options = options,
+            .prng = stdx.PRNG.from_seed(options.seed),
+            .files = files,
+        };
+    }
+
+    pub fn deinit(io: *IO) void {
+        for (io.files) |file| assert(file.closed);
+    }
+
+    /// Pass all queued submissions to the kernel and peek for completions.
+    pub fn run(io: *IO) !void {
+        while (io.completed.pop()) |completion| {
+            completion.callback(io, completion);
+        }
+    }
+
+    /// This struct holds the data needed for a single IO operation.
+    pub const Completion = struct {
+        link: Queue.Link,
+        context: ?*anyopaque,
+        callback: *const fn (*IO, *Completion) void,
+        operation: Operation,
+    };
+
+    const Operation = union(enum) {
+        read: struct {
+            fd: fd_t,
+            buf: [*]u8,
+            len: u32,
+            offset: u64,
+        },
+        write: struct {
+            fd: fd_t,
+            buf: [*]const u8,
+            len: u32,
+            offset: u64,
+        },
+        fsync: struct {
+            fd: fd_t,
+        },
+    };
+
+    fn submit(
+        self: *IO,
+        context: anytype,
+        comptime callback: anytype,
+        completion: *Completion,
+        comptime operation_tag: std.meta.Tag(Operation),
+        operation_data: std.meta.TagPayload(Operation, operation_tag),
+        comptime OperationImpl: type,
+    ) void {
+        const on_complete_fn = struct {
+            fn on_complete(io: *IO, _completion: *Completion) void {
+                // Perform the actual operation.
+                const op_data = &@field(_completion.operation, @tagName(operation_tag));
+                const result = OperationImpl.do_operation(io, op_data);
+
+                // Complete the Completion.
+                return callback(
+                    @ptrCast(@alignCast(_completion.context)),
+                    _completion,
+                    result,
+                );
+            }
+        }.on_complete;
+
+        completion.* = .{
+            .link = .{},
+            .context = context,
+            .callback = on_complete_fn,
+            .operation = @unionInit(Operation, @tagName(operation_tag), operation_data),
+        };
+
+        self.completed.push(completion);
+    }
+
+    pub const OpenDataFilePurpose = enum { format, open, inspect };
+
+    pub const ReadError = error{
+        WouldBlock,
+        NotOpenForReading,
+        ConnectionResetByPeer,
+        Alignment,
+        InputOutput,
+        IsDir,
+        SystemResources,
+        Unseekable,
+        ConnectionTimedOut,
+    } || posix.UnexpectedError;
+
+    pub fn read(
+        self: *IO,
+        comptime Context: type,
+        context: Context,
+        comptime callback: fn (
+            context: Context,
+            completion: *Completion,
+            result: ReadError!usize,
+        ) void,
+        completion: *Completion,
+        fd: fd_t,
+        buffer: []u8,
+        offset: u64,
+    ) void {
+        assert(fd < self.files.len);
+
+        self.submit(
+            context,
+            callback,
+            completion,
+            .read,
+            .{
+                .fd = fd,
+                .buf = buffer.ptr,
+                .len = @as(u32, @intCast(buffer_limit(buffer.len))),
+                .offset = offset,
+            },
+            struct {
+                fn do_operation(io: *IO, op: anytype) ReadError!usize {
+                    const sector_marked_in_fault_map = if (io.files[op.fd].fault_map) |fault_map|
+                        std.mem.readPackedIntNative(
+                            u1,
+                            fault_map,
+                            @divExact(op.offset, constants.sector_size),
+                        ) != 0
+                    else
+                        false;
+
+                    const sector_has_larger_than_logical_sector_read_fault =
+                        (op.len > constants.sector_size and io.prng.chance(
+                            io.options.larger_than_logical_sector_read_fault_probability,
+                        ));
+
+                    if (sector_marked_in_fault_map or
+                        sector_has_larger_than_logical_sector_read_fault)
+                    {
+                        return error.InputOutput;
+                    }
+
+                    const data = io.files[op.fd].buffer;
+                    stdx.copy_disjoint(.exact, u8, op.buf[0..op.len], data[op.offset..][0..op.len]);
+                    return op.len;
+                }
+            },
+        );
+    }
+
+    pub const WriteError = posix.PWriteError;
+
+    pub fn write(
+        self: *IO,
+        comptime Context: type,
+        context: Context,
+        comptime callback: fn (
+            context: Context,
+            completion: *Completion,
+            result: WriteError!usize,
+        ) void,
+        completion: *Completion,
+        fd: fd_t,
+        buffer: []const u8,
+        offset: u64,
+    ) void {
+        assert(fd < self.files.len);
+
+        self.submit(
+            context,
+            callback,
+            completion,
+            .write,
+            .{
+                .fd = fd,
+                .buf = buffer.ptr,
+                .len = @as(u32, @intCast(buffer_limit(buffer.len))),
+                .offset = offset,
+            },
+            struct {
+                fn do_operation(io: *IO, op: anytype) WriteError!usize {
+                    const data = io.files[op.fd].buffer;
+                    if (op.offset + op.len >= data.len) {
+                        @panic("write beyond simulated file size");
+                    }
+                    stdx.copy_disjoint(.exact, u8, data[op.offset..][0..op.len], op.buf[0..op.len]);
+                    return op.len;
+                }
+            },
+        );
+    }
+
+    pub const FsyncError = posix.SyncError;
+
+    pub fn fsync(
+        self: *IO,
+        comptime Context: type,
+        context: Context,
+        comptime callback: fn (
+            context: Context,
+            completion: *Completion,
+            result: FsyncError!void,
+        ) void,
+        completion: *Completion,
+        fd: fd_t,
+    ) void {
+        assert(fd < self.files.len);
+
+        self.submit(
+            context,
+            callback,
+            completion,
+            .fsync,
+            .{ .fd = fd },
+            struct {
+                fn do_operation(_: *IO, _: anytype) FsyncError!void {}
+            },
+        );
+    }
+
+    pub fn aof_blocking_write_all(self: *IO, fd: fd_t, source: []const u8) posix.WriteError!void {
+        assert(fd < self.files.len);
+
+        const file_index = @as(u32, @intCast(fd));
+        const file = &self.files[file_index];
+        const target = file.buffer;
+        const offset = file.offset;
+
+        assert(offset + source.len <= target.len);
+
+        stdx.copy_disjoint(.exact, u8, target[offset..][0..source.len], source);
+
+        file.offset += @as(u32, @intCast(source.len));
+    }
+
+    pub const PReadError = posix.PReadError;
+
+    pub fn aof_blocking_close(self: *IO, fd: fd_t) void {
+        assert(fd < self.files.len);
+        self.files[fd].closed = true;
+    }
+
+    pub fn aof_blocking_pread_all(self: *IO, fd: fd_t, target: []u8, offset: u64) PReadError!usize {
+        assert(fd < self.files.len);
+
+        const file_index = @as(u32, @intCast(fd));
+        const source = self.files[file_index].buffer;
+
+        assert(offset + target.len <= source.len);
+
+        stdx.copy_disjoint(.exact, u8, target, source[offset..][0..target.len]);
+
+        return target.len;
+    }
+
+    pub fn aof_blocking_stat(_: *IO, _: []const u8) std.fs.Dir.StatFileError!std.fs.File.Stat {
+        return error.Unexpected;
+    }
+
+    pub fn aof_blocking_fstat(_: *IO, _: fd_t) std.fs.Dir.StatError!std.fs.File.Stat {
+        return error.Unexpected;
+    }
+
+    pub fn aof_blocking_open(_: *IO, _: []const u8) !fd_t {
+        return error.Unexpected;
+    }
+
+    pub fn reset(self: *IO) void {
+        self.completed.reset();
+    }
+};

data/ext/tb_client/tigerbeetle/src/testing/marks.zig

@@ -0,0 +1,126 @@
+//! This file piggy-backs on the logging infrastructure to implement explicit coverage marks:
+//!     <https://ferrous-systems.com/blog/coverage-marks/>
+//!
+//! In production code, you can mark certain log lines as "this should be covered by a test".
+//! In test code, you can then assert that a _specific_ test covers a specific log line. The two
+//! benefits are:
+//! - tests are more resilient to refactors
+//! - production code is more readable (you can immediately jump to a specific test)
+//!
+//! At the surface level, this resembles usual code coverage, but the idea is closer to traceability
+//! from safety-critical systems:
+//!     <https://en.wikipedia.org/wiki/Requirements_traceability>
+//!
+//! That is, the important part is not that a log line is covered at all, but that we can trace
+//! production code to a single minimal hand-written test which explains why the code needs to
+//! exist.
+test "tutorial" {
+    // Import by a qualified name.
+    const marks = @import("./marks.zig");
+
+    const production_code = struct {
+        // In production code, wrap the logger.
+        const log = marks.wrap_log(std.log.scoped(.my_module));
+
+        fn function_under_test(x: u32) void {
+            if (x % 2 == 0) {
+                // Both `log.info` and log.covered.info` are available.
+                // Only second version records coverage.
+                log.mark.info("x is even (x={})", .{x});
+            }
+        }
+    };
+
+    // Create a mark with the `mark` function...
+    const mark = marks.check("x is even");
+    production_code.function_under_test(92);
+    try mark.expect_hit(); // ... and don't forget to assert at the end!
+}
+
+const std = @import("std");
+const assert = std.debug.assert;
+const builtin = @import("builtin");
+
+const GlobalStateType = if (builtin.is_test) struct {
+    mark_name: ?[]const u8 = null,
+    mark_hit_count: u32 = 0,
+} else void;
+
+/// Stores the currently active mark and its hit count. State is not synchronized and assumes
+/// single threaded execution.
+var global_state: GlobalStateType = .{};
+
+pub const Mark = struct {
+    name: []const u8,
+
+    pub fn expect_hit(mark: Mark) !void {
+        comptime assert(builtin.is_test);
+        assert(global_state.mark_name.?.ptr == mark.name.ptr);
+        defer global_state = .{};
+
+        if (global_state.mark_hit_count == 0) {
+            std.debug.print("mark '{s}' not hit", .{mark.name});
+            return error.MarkNotHit;
+        }
+    }
+
+    pub fn expect_not_hit(mark: Mark) !void {
+        comptime assert(builtin.is_test);
+        assert(global_state.mark_name.?.ptr == mark.name.ptr);
+        defer global_state = .{};
+
+        if (global_state.mark_hit_count != 0) {
+            std.debug.print("mark '{s}' hit", .{mark.name});
+            return error.MarkHit;
+        }
+    }
+};
+
+pub fn check(name: []const u8) Mark {
+    comptime assert(builtin.is_test);
+    assert(global_state.mark_name == null);
+    assert(global_state.mark_hit_count == 0);
+
+    global_state.mark_name = name;
+    return Mark{ .name = name };
+}
+
+pub fn wrap_log(comptime base: type) type {
+    return struct {
+        pub const mark = if (builtin.is_test) struct {
+            pub fn err(comptime fmt: []const u8, args: anytype) void {
+                record(fmt);
+                base.err(fmt, args);
+            }
+
+            pub fn warn(comptime fmt: []const u8, args: anytype) void {
+                record(fmt);
+                base.warn(fmt, args);
+            }
+
+            pub fn info(comptime fmt: []const u8, args: anytype) void {
+                record(fmt);
+                base.info(fmt, args);
+            }
+
+            pub fn debug(comptime fmt: []const u8, args: anytype) void {
+                record(fmt);
+                base.debug(fmt, args);
+            }
+        } else base;
+
+        pub const err = base.err;
+        pub const warn = base.warn;
+        pub const info = base.info;
+        pub const debug = base.debug;
+    };
+}
+
+fn record(fmt: []const u8) void {
+    comptime assert(builtin.is_test);
+    if (global_state.mark_name) |mark_active| {
+        if (std.mem.indexOf(u8, fmt, mark_active) != null) {
+            global_state.mark_hit_count += 1;
+        }
+    }
+}