tigerbeetle 0.0.36 → 0.0.38

data/ext/tb_client/tigerbeetle/src/vsr/replica_format.zig

@@ -0,0 +1,416 @@
+const std = @import("std");
+const assert = std.debug.assert;
+
+const constants = @import("../constants.zig");
+const stdx = @import("stdx");
+const vsr = @import("../vsr.zig");
+const Header = vsr.Header;
+const data_file_size_min = @import("./superblock.zig").data_file_size_min;
+
+/// Initialize the TigerBeetle replica's data file.
+pub fn format(
+    comptime Storage: type,
+    gpa: std.mem.Allocator,
+    storage: *Storage,
+    options: vsr.SuperBlockType(Storage).FormatOptions,
+) !void {
+    const ReplicaFormat = ReplicaFormatType(Storage);
+    const SuperBlock = vsr.SuperBlockType(Storage);
+
+    var superblock = try SuperBlock.init(gpa, storage, .{
+        .storage_size_limit = data_file_size_min,
+    });
+    defer superblock.deinit(gpa);
+
+    var replica_format = try ReplicaFormat.init(gpa);
+    defer replica_format.deinit(gpa);
+
+    try replica_format.queue_format_wal(options.cluster, storage);
+    replica_format.format_and_tick(storage, &superblock, options);
+    replica_format.verify_writes();
+}
+
+/// When formatting, we write:
+/// * constants.journal_slot_count many prepares,
+/// * 1 write that contains all of the headers.
+pub const writes_max = constants.journal_slot_count + 1;
+
+fn ReplicaFormatType(comptime Storage: type) type {
+    const SuperBlock = vsr.SuperBlockType(Storage);
+    return struct {
+        const ReplicaFormat = @This();
+
+        const Write = struct {
+            write: Storage.Write = undefined,
+            replica_format: *ReplicaFormat,
+
+            issued_buffer: []const u8,
+            issued_offset: u64,
+        };
+
+        formatting: bool = false,
+        formatting_superblock: bool = false,
+        superblock_context: SuperBlock.Context = undefined,
+
+        writes: [writes_max]Write = undefined,
+        writes_pending: u64 = 0,
+
+        sectors_written: std.DynamicBitSetUnmanaged,
+        arena: std.heap.ArenaAllocator,
+
+        fn init(gpa: std.mem.Allocator) !ReplicaFormat {
+            var sectors_written = try std.DynamicBitSetUnmanaged.initEmpty(
+                gpa,
+                @divExact(data_file_size_min, constants.sector_size),
+            );
+            errdefer sectors_written.deinit(gpa);
+
+            var arena = std.heap.ArenaAllocator.init(gpa);
+            errdefer arena.deinit();
+
+            return .{
+                .sectors_written = sectors_written,
+                .arena = arena,
+            };
+        }
+
+        fn deinit(self: *ReplicaFormat, gpa: std.mem.Allocator) void {
+            self.arena.deinit();
+            self.sectors_written.deinit(gpa);
+        }
+
+        fn queue_format_wal(
+            self: *ReplicaFormat,
+            cluster: u128,
+            storage: *Storage,
+        ) !void {
+            assert(!self.formatting and !self.formatting_superblock);
+
+            const arena = self.arena.allocator();
+
+            // The logical offset *within the Zone*.
+            // Even though the prepare zone follows the redundant header zone, write the prepares
+            // first. This allows the test Storage to check the invariant "never write the redundant
+            // header before the prepare".
+            for (0..constants.journal_slot_count) |slot| {
+                // Direct I/O requires the buffer to be sector-aligned. Allocate a buffer for each
+                // sector in the arena, so they can be written concurrently.
+                const header_buffer = try arena.alignedAlloc(
+                    u8,
+                    constants.sector_size,
+                    constants.sector_size,
+                );
+                const header: *Header.Prepare = std.mem.bytesAsValue(
+                    Header.Prepare,
+                    header_buffer,
+                );
+                header.* = slot_header(cluster, slot);
+                assert(header.valid_checksum());
+
+                const prepare_offset = slot * constants.message_size_max;
+                assert(prepare_offset <= constants.journal_size_prepares);
+                assert(prepare_offset % @sizeOf(Header) == 0);
+                assert(prepare_offset % constants.sector_size == 0);
+
+                // Zero padding to produce identical checksums of an empty datafile, not because
+                // it's required for correctness.
+                const header_padding = header_buffer[@sizeOf(Header.Prepare)..];
+                @memset(header_padding, 0);
+                assert(stdx.zeroed(header_padding));
+
+                if (header.op == 0) {
+                    assert(header.operation == .root);
+                } else {
+                    assert(header.operation == .reserved);
+                }
+
+                self.writes[self.writes_pending] = .{
+                    .replica_format = self,
+                    .issued_buffer = header_buffer,
+                    .issued_offset = prepare_offset,
+                };
+
+                storage.write_sectors(
+                    write_sectors_callback,
+                    &self.writes[self.writes_pending].write,
+                    header_buffer,
+                    .wal_prepares,
+                    prepare_offset,
+                );
+                self.writes_pending += 1;
+            }
+
+            // Direct I/O requires the buffer to be sector-aligned. Unlike the Prepares above that
+            // require a buffer per prepare, since they are spread out with zeros inbetween, the
+            // headers zone is contiguous so a single buffer will do.
+            //
+            // There might be padding, so allocate []u8 instead of []Header.Prepare.
+            const headers_buffer = try arena.alignedAlloc(
+                u8,
+                constants.sector_size,
+                vsr.sector_ceil(constants.journal_size_headers),
+            );
+
+            for (0..constants.journal_slot_count) |slot| {
+                const header_buffer =
+                    headers_buffer[slot * @sizeOf(Header.Prepare) ..][0..@sizeOf(Header.Prepare)];
+                const header: *Header.Prepare = @alignCast(
+                    std.mem.bytesAsValue(Header.Prepare, header_buffer),
+                );
+                header.* = slot_header(cluster, slot);
+                assert(header.valid_checksum());
+
+                if (header.op == 0) {
+                    assert(header.operation == .root);
+                } else {
+                    assert(header.operation == .reserved);
+                }
+            }
+
+            // Zero padding to produce identical checksums of an empty datafile, not because it's
+            // required for correctness.
+            const headers_padding =
+                headers_buffer[constants.journal_slot_count * @sizeOf(Header.Prepare) ..];
+            @memset(headers_padding, 0);
+            assert(stdx.zeroed(headers_padding));
+
+            self.writes[self.writes_pending] = .{
+                .replica_format = self,
+                .issued_buffer = headers_buffer,
+                .issued_offset = 0,
+            };
+            storage.write_sectors(
+                write_sectors_callback,
+                &self.writes[self.writes_pending].write,
+                headers_buffer,
+                .wal_headers,
+                0,
+            );
+            self.writes_pending += 1;
+        }
+
+        fn format_and_tick(
+            self: *ReplicaFormat,
+            storage: *Storage,
+            superblock: *SuperBlock,
+            superblock_options: SuperBlock.FormatOptions,
+        ) void {
+            assert(self.writes_pending == writes_max);
+
+            self.formatting = true;
+            while (self.formatting) storage.run();
+
+            self.formatting_superblock = true;
+            superblock.format(
+                format_superblock_callback,
+                &self.superblock_context,
+                superblock_options,
+            );
+            while (self.formatting_superblock) storage.run();
+        }
+
+        fn write_sectors_callback(storage_write: *Storage.Write) void {
+            const write: *Write = @fieldParentPtr("write", storage_write);
+            const self = write.replica_format;
+
+            assert(self.formatting);
+            assert(!self.formatting_superblock);
+
+            self.writes_pending -= 1;
+
+            const sector_offset = @divExact(
+                storage_write.zone.offset(write.issued_offset),
+                constants.sector_size,
+            );
+            const sector_count = @divExact(write.issued_buffer.len, constants.sector_size);
+
+            for (sector_offset..sector_offset + sector_count) |sector| {
+                self.sectors_written.set(sector);
+            }
+
+            if (self.writes_pending == 0) {
+                self.formatting = false;
+            }
+        }
+
+        fn format_superblock_callback(superblock_context: *SuperBlock.Context) void {
+            const self: *ReplicaFormat =
+                @alignCast(@fieldParentPtr("superblock_context", superblock_context));
+            assert(!self.formatting);
+            assert(self.formatting_superblock);
+            self.formatting_superblock = false;
+        }
+
+        fn verify_writes(self: *ReplicaFormat) void {
+            assert(!self.formatting and !self.formatting_superblock);
+            assert(self.writes_pending == 0);
+
+            assert(self.sectors_written.count() > 0);
+            assert(self.sectors_written.capacity() ==
+                @divExact(data_file_size_min, constants.sector_size));
+
+            // Expect that:
+            // * every sector in the wal_headers zone has been written,
+            // * the first sector in every constants.message_size_max has been written,
+            // * nothing else has been written.
+            //
+            // This might seem to miss the superblock zone, but that's handled entirely by
+            // superblock.zig, which reads back the headers to validate it has been written
+            // correctly.
+            for (0..self.sectors_written.capacity()) |sector| {
+                const sector_start = sector * constants.sector_size;
+
+                const zone = for (std.enums.values(vsr.Zone)) |zone| {
+                    if (sector_start >= zone.start() and
+                        sector_start < zone.start() + zone.size().?) break zone;
+                } else unreachable;
+
+                switch (zone) {
+                    // Every sector in the wal_headers zone has been written:
+                    .wal_headers => assert(self.sectors_written.isSet(sector)),
+
+                    // The first sector in every constants.message_size_max has been written:
+                    .wal_prepares => {
+                        if ((sector_start - zone.start()) % constants.message_size_max == 0) {
+                            assert(self.sectors_written.isSet(sector));
+                        } else {
+                            assert(!self.sectors_written.isSet(sector));
+                        }
+                    },
+
+                    // Nothing else has been written:
+                    else => assert(!self.sectors_written.isSet(sector)),
+                }
+            }
+        }
+    };
+}
+
+pub fn slot_header(cluster: u128, slot: u64) Header.Prepare {
+    assert(slot < constants.journal_slot_count);
+    assert(slot * @sizeOf(Header.Prepare) < constants.journal_size_headers);
+    assert(slot * constants.message_size_max < constants.journal_size_prepares);
+    assert(@sizeOf(Header.Prepare) < constants.sector_size);
+
+    return if (slot == 0)
+        Header.Prepare.root(cluster)
+    else
+        Header.Prepare.reserve(cluster, slot);
+}
+
+test slot_header {
+    const allocator = std.testing.allocator;
+
+    const header_buffer = try allocator.create(Header.Prepare);
+    defer allocator.destroy(header_buffer);
+
+    for (0..constants.journal_slot_count) |slot| {
+        const header = slot_header(0, slot);
+
+        try std.testing.expect(header.valid_checksum());
+        try std.testing.expect(header.valid_checksum_body(&[0]u8{}));
+        try std.testing.expectEqual(header.invalid(), null);
+        try std.testing.expectEqual(header.cluster, 0);
+        try std.testing.expectEqual(header.op, slot);
+        try std.testing.expectEqual(header.size, @sizeOf(vsr.Header));
+        try std.testing.expectEqual(header.command, .prepare);
+        if (slot == 0) {
+            try std.testing.expectEqual(header.operation, .root);
+        } else {
+            try std.testing.expectEqual(header.operation, .reserved);
+        }
+    }
+}
+
+test "format" {
+    const Storage = @import("../testing/storage.zig").Storage;
+    const fixtures = @import("../testing/fixtures.zig");
+    const allocator = std.testing.allocator;
+    const cluster = 0;
+    const replica = 1;
+    const replica_count = 1;
+
+    var storage = try fixtures.init_storage(allocator, .{
+        .size = data_file_size_min,
+        .iops_write_max = writes_max,
+    });
+    defer storage.deinit(allocator);
+
+    try format(Storage, allocator, &storage, .{
+        .cluster = cluster,
+        .release = vsr.Release.minimum,
+        .replica = replica,
+        .replica_count = replica_count,
+        .view = null,
+    });
+
+    // Verify the superblock headers.
+    var copy: u8 = 0;
+    while (copy < constants.superblock_copies) : (copy += 1) {
+        const superblock_header = storage.superblock_header(copy);
+
+        try std.testing.expectEqual(superblock_header.copy, copy);
+        try std.testing.expectEqual(superblock_header.cluster, cluster);
+        try std.testing.expectEqual(superblock_header.sequence, 1);
+        try std.testing.expectEqual(
+            superblock_header.vsr_state.checkpoint.storage_size,
+            storage.size,
+        );
+        try std.testing.expectEqual(superblock_header.vsr_state.checkpoint.header.op, 0);
+        try std.testing.expectEqual(superblock_header.vsr_state.commit_max, 0);
+        try std.testing.expectEqual(superblock_header.vsr_state.view, 0);
+        try std.testing.expectEqual(superblock_header.vsr_state.log_view, 0);
+        try std.testing.expectEqual(
+            superblock_header.vsr_state.replica_id,
+            superblock_header.vsr_state.members[replica],
+        );
+        try std.testing.expectEqual(superblock_header.vsr_state.replica_count, replica_count);
+    }
+
+    // Verify the WAL headers and prepares zones.
+    for (storage.wal_headers(), storage.wal_prepares(), 0..) |header, *message, slot| {
+        try std.testing.expect(std.meta.eql(header, message.header));
+
+        try std.testing.expect(header.valid_checksum());
+        try std.testing.expect(header.valid_checksum_body(&[0]u8{}));
+        try std.testing.expectEqual(header.invalid(), null);
+        try std.testing.expectEqual(header.cluster, cluster);
+        try std.testing.expectEqual(header.op, slot);
+        try std.testing.expectEqual(header.size, @sizeOf(vsr.Header));
+        try std.testing.expectEqual(header.command, .prepare);
+        if (slot == 0) {
+            try std.testing.expectEqual(header.operation, .root);
+        } else {
+            try std.testing.expectEqual(header.operation, .reserved);
+        }
+    }
+
+    // Verify client replies. The contents are not zeroed.
+    try std.testing.expectEqual(storage.client_replies().len, constants.clients_max);
+
+    // Verify grid alignment. The contents of the padding are not zeroed.
+    try std.testing.expect(vsr.Zone.grid.start() % constants.sector_size == 0);
+
+    // Explicitly zero client_replies and the grid padding. This is not required for formatting, but
+    // it allows for easy checksums of the entire testing storage.
+    @memset(
+        storage.memory[vsr.Zone.client_replies.offset(0)..][0..vsr.Zone.client_replies.size().?],
+        0,
+    );
+    if (vsr.Zone.grid_padding.size().? > 0) {
+        @memset(
+            storage.memory[vsr.Zone.grid_padding.offset(0)..][0..vsr.Zone.grid_padding.size().?],
+            0,
+        );
+    }
+
+    // Lastly, verify the entire storage contents against a known good checksum for the given
+    // cluster, replica and replica count.
+    //
+    // This doesn't match the output from `tigerbeetle format ...` since the testing storage / slot
+    // counts are lower.
+    try std.testing.expectEqual(
+        vsr.checksum(storage.memory),
+        339529914272821912685300045374558551362,
+    );
+}

data/ext/tb_client/tigerbeetle/src/vsr/replica_reformat.zig

@@ -0,0 +1,165 @@
+//! Replica recovery: Format a data file to replace one which was permanently lost.
+//!
+//! 1. The recovery process send `pipeline_prepare_queue_max` requests (1 register + many noops) to
+//!    the cluster.
+//! 2. Once those have committed, it creates the new data file. The data file is identical to
+//!    `tigerbeetle format`'s output *except* that `vsr_state.view == client.view + 2` (where
+//!    `client.view` is the view number of the client at the end of committing the requests).
+//! 3. The recovery process exits. Now running `tigerbeetle start` as normal will work.
+//!
+//! The `pipeline_prepare_queue_max` committed requests ensure that if the newly recovered replica
+//! nacks uncommitted ops via a DVC message, it is nacking ops which were definitely not received by
+//! the previous version of the replica.
+//!
+//! The +2 is because:
+//! - We don't want to join in the same view, since the replica might have participated in it before
+//!   being lost, and we can't remember any promises we made.
+//! - Likewise, we don't want to go to view + 1 -- if we were the first to collect a SVC quorum
+//!   before being lost, we might have sent a DVC. Since we don't remember, we must skip past
+//!   `view + 11 to ensure that we don't send a different DVC. (We have the invariant that if a
+//!   replica sends a DVC for a given view, then all DVC's it sends for that view will be
+//!   identical.)
+const std = @import("std");
+const assert = std.debug.assert;
+
+const constants = @import("../constants.zig");
+const vsr = @import("../vsr.zig");
+const format = @import("./replica_format.zig").format;
+
+const log = std.log.scoped(.reformat);
+
+pub fn ReplicaReformatType(
+    comptime StateMachine: type,
+    comptime MessageBus: type,
+    comptime Storage: type,
+) type {
+    const Client = vsr.ClientType(StateMachine.Operation, MessageBus);
+    const SuperBlock = vsr.SuperBlockType(Storage);
+
+    return struct {
+        const ReplicaReformat = @This();
+
+        const Result = union(enum) {
+            failed: anyerror,
+            ok,
+        };
+
+        allocator: std.mem.Allocator,
+        options: SuperBlock.FormatOptions,
+        client: *Client,
+        storage: *Storage,
+
+        requests_done: u32 = 0,
+        result: ?Result = null,
+
+        pub fn init(
+            allocator: std.mem.Allocator,
+            client: *Client,
+            storage: *Storage,
+            options: SuperBlock.FormatOptions,
+        ) !ReplicaReformat {
+            assert(options.view == null);
+            assert(options.replica_count >= 3);
+
+            return .{
+                .allocator = allocator,
+                .options = options,
+                .client = client,
+                .storage = storage,
+            };
+        }
+
+        pub fn deinit(reformat: *ReplicaReformat, allocator: std.mem.Allocator) void {
+            _ = reformat;
+            _ = allocator;
+        }
+        pub fn done(reformat: *const ReplicaReformat) ?Result {
+            assert(reformat.requests_done <= constants.pipeline_prepare_queue_max);
+            return reformat.result;
+        }
+
+        pub fn start(reformat: *ReplicaReformat) void {
+            assert(reformat.requests_done == 0);
+            const user_data = @intFromPtr(reformat);
+            reformat.client.register(client_register_callback, user_data);
+        }
+
+        fn client_register_callback(
+            user_data: u128,
+            register_result: *const vsr.RegisterResult,
+        ) void {
+            _ = register_result;
+            const reformat: *ReplicaReformat = @ptrFromInt(@as(usize, @intCast(user_data)));
+            assert(reformat.requests_done == 0);
+
+            log.debug("{}: register", .{reformat.options.replica});
+
+            reformat.requests_done += 1;
+            reformat.client_request();
+        }
+
+        fn client_request(reformat: *ReplicaReformat) void {
+            assert(reformat.requests_done < constants.pipeline_prepare_queue_max);
+
+            log.debug("{}: request start={}", .{
+                reformat.options.replica,
+                reformat.requests_done,
+            });
+
+            const message = reformat.client.get_message().build(.request);
+            errdefer reformat.client.release_message(message.base());
+
+            message.header.* = .{
+                .client = reformat.client.id,
+                .request = 0, // Set inside `raw_request`.
+                .cluster = reformat.client.cluster,
+                .command = .request,
+                .release = reformat.client.release,
+                .operation = .noop,
+                .size = @sizeOf(vsr.Header),
+                .previous_request_latency = 0,
+            };
+
+            const user_data = @intFromPtr(reformat);
+            reformat.client.raw_request(client_request_callback, user_data, message);
+        }
+
+        fn client_request_callback(
+            user_data: u128,
+            operation: vsr.Operation,
+            timestamp: u64,
+            results: []u8,
+        ) void {
+            assert(operation == .noop);
+            assert(timestamp > 0);
+
+            const reformat: *ReplicaReformat = @ptrFromInt(@as(usize, @intCast(user_data)));
+            assert(reformat.requests_done > 0);
+            assert(reformat.requests_done < constants.pipeline_prepare_queue_max);
+            assert(results.len == 0);
+
+            log.debug("{}: request done={}", .{
+                reformat.options.replica,
+                reformat.requests_done,
+            });
+
+            reformat.requests_done += 1;
+            if (reformat.requests_done == constants.pipeline_prepare_queue_max) {
+                // +2 since we might have sent a DVC as part of +1 before we crashed.
+                reformat.options.view = reformat.client.view + 2;
+                format(
+                    Storage,
+                    reformat.allocator,
+                    reformat.storage,
+                    reformat.options,
+                ) catch |err| {
+                    reformat.result = .{ .failed = err };
+                    return;
+                };
+                reformat.result = .ok;
+            } else {
+                reformat.client_request();
+            }
+        }
+    };
+}