tigerbeetle 0.0.38 → 0.0.39

data/ext/tb_client/tigerbeetle/src/stdx/huge_page_allocator.zig

@@ -0,0 +1,115 @@
+const std = @import("std");
+const builtin = @import("builtin");
+const stdx = @import("stdx.zig");
+const Allocator = std.mem.Allocator;
+
+const log = std.log.scoped(.allocator);
+
+const page_allocator_vtable = std.heap.page_allocator.vtable;
+
+/// Like `std.heap.page_allocator`, but on Linux applies `MADV_HUGEPAGE` to
+/// allocated regions so that the kernel may back them with 2 MiB transparent
+/// huge pages, reducing TLB pressure for large allocations.
+///
+/// Only `alloc` is intercepted. `resize` and `remap` inherit the VMA flags
+/// (including `VM_HUGEPAGE`) set on the original mapping, so they need no
+/// additional `madvise` call.
+///
+/// On non-Linux targets this is identical to `std.heap.page_allocator`.
+pub const huge_page_allocator: Allocator = .{
+    .ptr = std.heap.page_allocator.ptr,
+    .vtable = if (builtin.target.os.tag == .linux) &vtable else page_allocator_vtable,
+};
+
+const vtable: Allocator.VTable = .{
+    .alloc = alloc,
+    .resize = page_allocator_vtable.resize,
+    .remap = page_allocator_vtable.remap,
+    .free = page_allocator_vtable.free,
+};
+
+fn alloc(context: *anyopaque, n: usize, alignment: std.mem.Alignment, ra: usize) ?[*]u8 {
+    const ptr = page_allocator_vtable.alloc(context, n, alignment, ra) orelse return null;
+    // This is just a hint, so if it fails we can safely ignore it.
+    std.posix.madvise(@alignCast(ptr), n, std.posix.MADV.HUGEPAGE) catch {
+        log.warn("Transparent Huge Pages (THP) are disabled.", .{});
+    };
+    return ptr;
+}
+
+const testing = std.testing;
+const assert = std.debug.assert;
+
+/// Checks /proc/self/smaps for the "hg" VmFlag on the mapping containing `ptr`.
+fn verify_address_is_huge_page(ptr: [*]const u8) !bool {
+    assert(builtin.target.os.tag == .linux);
+    const addr = @intFromPtr(ptr);
+
+    var file = try std.fs.openFileAbsolute("/proc/self/smaps", .{});
+    defer file.close();
+
+    const content = try file.readToEndAlloc(testing.allocator, 10 * 1024 * 1024);
+    defer testing.allocator.free(content);
+
+    var lines = std.mem.splitScalar(u8, content, '\n');
+
+    // Find the mapping header that contains our address.
+    while (lines.next()) |line| {
+        if (parse_mapping_range(line)) |range| {
+            if (addr >= range.min and addr < range.max) {
+                // Scan subsequent lines for VmFlags within this mapping.
+                while (lines.next()) |detail| {
+                    if (stdx.cut_prefix(detail, "VmFlags:")) |rest| {
+                        return stdx.cut(rest, "hg") != null;
+                    }
+                }
+            }
+        }
+    }
+    return false;
+}
+
+fn parse_mapping_range(line: []const u8) ?struct { min: u64, max: u64 } {
+    const addr_range, _ = stdx.cut(line, " ") orelse return null;
+    const addr_hex_min, const addr_hex_max = stdx.cut(addr_range, "-") orelse return null;
+    const addr_min = std.fmt.parseUnsigned(u64, addr_hex_min, 16) catch return null;
+    const addr_max = std.fmt.parseUnsigned(u64, addr_hex_max, 16) catch return null;
+    return .{ .min = addr_min, .max = addr_max };
+}
+
+test "huge_page_allocator: basic alloc and free" {
+    const slice = try huge_page_allocator.alloc(u8, 4096);
+    defer huge_page_allocator.free(slice);
+
+    @memset(slice, 0xab);
+    try testing.expectEqual(@as(u8, 0xab), slice[0]);
+}
+
+test "huge_page_allocator: large THP-eligible allocation" {
+    // 4 MiB — large enough for THP promotion on Linux.
+    const size = 4 * 1024 * 1024;
+    const slice = try huge_page_allocator.alloc(u8, size);
+    defer huge_page_allocator.free(slice);
+
+    @memset(slice, 0xcd);
+    try testing.expectEqual(@as(u8, 0xcd), slice[size - 1]);
+
+    if (builtin.target.os.tag == .linux) {
+        // Verify that MADV_HUGEPAGE was applied by checking VmFlags in /proc/self/smaps.
+        // The "hg" flag means the process requested hugepages via madvise — this is
+        // deterministic regardless of whether the kernel actually promoted the pages.
+        try testing.expect(try verify_address_is_huge_page(slice.ptr));
+    }
+}
+
+test "huge_page_allocator: as ArenaAllocator backing" {
+    var arena = std.heap.ArenaAllocator.init(huge_page_allocator);
+    defer arena.deinit();
+
+    const alloc1 = try arena.allocator().alloc(u8, 1024);
+    const alloc2 = try arena.allocator().alloc(u8, 2048);
+    @memset(alloc1, 1);
+    @memset(alloc2, 2);
+    try testing.expectEqual(@as(u8, 1), alloc1[0]);
+    try testing.expectEqual(@as(u8, 2), alloc2[0]);
+}

data/ext/tb_client/tigerbeetle/src/stdx/stdx.zig

@@ -16,6 +16,8 @@ pub const Snap = @import("testing/snaptest.zig").Snap;
 pub const ZipfianGenerator = @import("zipfian.zig").ZipfianGenerator;
 pub const ZipfianShuffled = @import("zipfian.zig").ZipfianShuffled;
 
+pub const huge_page_allocator = @import("huge_page_allocator.zig").huge_page_allocator;
+
 pub const aegis = @import("vendored/aegis.zig");
 pub const dbg = @import("debug.zig").dbg;
 pub const flags = @import("flags.zig").parse;
@@ -1144,6 +1146,7 @@ pub fn term_from_status(status: u32) std.process.Child.Term {
 }
 
 comptime {
+    _ = @import("huge_page_allocator.zig");
     _ = @import("vendored/aegis.zig");
     _ = @import("bit_set.zig");
     _ = @import("bounded_array.zig");

data/ext/tb_client/tigerbeetle/src/testing/vortex/constants.zig

@@ -5,10 +5,10 @@ pub const vsr = @import("../../constants.zig");
 
 pub const vortex = struct {
     pub const cluster_id = 0;
-    pub const connections_count_max = @divFloor(
-        constants.vsr.clients_max,
-        constants.vsr.replicas_max,
-    );
+    // Maximum number of connections *per replica*.
+    // -1 since replicas don't connect to themselves.
+    // +1 for the single driver/client.
+    pub const connections_count_max = (vsr.replicas_max - 1) + 1;
 
     // We allow the cluster to not make progress processing requests for this amount of time.
     // After that it's considered a test failure.

data/ext/tb_client/tigerbeetle/src/testing/vortex/faulty_network.zig

@@ -34,19 +34,17 @@ const Faults = struct {
     };
 
     delay: ?Delay = null,
-    lose: ?Ratio = null,
     corrupt: ?Ratio = null,
 
     // Others not implemented: duplication, reordering, rate
 
     pub fn heal(faults: *Faults) void {
         faults.delay = null;
-        faults.lose = null;
         faults.corrupt = null;
     }
 
     pub fn is_healed(faults: *const Faults) bool {
-        return faults.delay == null and faults.lose == null and faults.corrupt == null;
+        return faults.delay == null and faults.corrupt == null;
     }
 };
 
@@ -70,11 +68,13 @@ const Pipe = struct {
     ) void {
         assert(pipe.connection.state == .proxying);
         assert(pipe.status == .idle);
+        assert(pipe.input == null);
+        assert(pipe.output == null);
+        assert(pipe.recv_size == 0);
+        assert(pipe.send_size == 0);
 
         pipe.input = input;
         pipe.output = output;
-        pipe.recv_size = 0;
-        pipe.send_size = 0;
 
         // Kick off the recv/send loop.
         pipe.recv();
@@ -94,16 +94,19 @@ const Pipe = struct {
         pipe.connection.io.recv(
             *Pipe,
             pipe,
-            on_recv,
+            recv_callback,
             &pipe.recv_completion,
             pipe.input.?,
             pipe.buffer[0..],
         );
     }
 
-    fn on_recv(pipe: *Pipe, _: *IO.Completion, result: IO.RecvError!usize) void {
+    fn recv_callback(pipe: *Pipe, _: *IO.Completion, result: IO.RecvError!usize) void {
         assert(pipe.recv_size == 0);
         assert(pipe.send_size == 0);
+        assert(pipe.connection.state != .free);
+        assert(pipe.connection.state != .accepting);
+        assert(pipe.connection.state != .connecting);
 
         assert(pipe.status == .recv);
         pipe.status = .idle;
@@ -125,18 +128,6 @@ const Pipe = struct {
             return pipe.connection.try_close();
         }
 
-        if (pipe.connection.network.faults.lose) |lose| {
-            if (pipe.connection.network.prng.chance(lose)) {
-                log.debug("losing {d} bytes ({d},{d})", .{
-                    pipe.recv_size,
-                    pipe.connection.replica_index,
-                    pipe.connection.connection_index,
-                });
-                pipe.recv();
-                return;
-            }
-        }
-
         if (pipe.connection.network.faults.corrupt) |corrupt| {
             if (pipe.connection.network.prng.chance(corrupt)) {
                 switch (pipe.connection.network.prng.enum_uniform(enum { shuffle, zero })) {
@@ -180,14 +171,23 @@ const Pipe = struct {
 
             assert(pipe.status == .idle);
             pipe.status = .send_timeout;
-            pipe.io.timeout(*Pipe, pipe, on_timeout, &pipe.send_completion, timeout_duration_ns);
+            pipe.io.timeout(
+                *Pipe,
+                pipe,
+                timeout_callback,
+                &pipe.send_completion,
+                timeout_duration_ns,
+            );
         } else {
             pipe.send();
         }
     }
 
-    fn on_timeout(pipe: *Pipe, _: *IO.Completion, result: IO.TimeoutError!void) void {
+    fn timeout_callback(pipe: *Pipe, _: *IO.Completion, result: IO.TimeoutError!void) void {
         assert(pipe.status == .send_timeout);
+        assert(pipe.connection.state != .free);
+        assert(pipe.connection.state != .accepting);
+        assert(pipe.connection.state != .connecting);
         pipe.status = .idle;
 
         if (pipe.connection.state != .proxying) return;
@@ -206,15 +206,18 @@ const Pipe = struct {
         pipe.io.send(
             *Pipe,
             pipe,
-            on_send,
+            send_callback,
             &pipe.send_completion,
             pipe.output.?,
             pipe.buffer[pipe.send_size..pipe.recv_size],
         );
     }
 
-    fn on_send(pipe: *Pipe, _: *IO.Completion, result: IO.SendError!usize) void {
+    fn send_callback(pipe: *Pipe, _: *IO.Completion, result: IO.SendError!usize) void {
         assert(pipe.send_size < pipe.recv_size);
+        assert(pipe.connection.state != .free);
+        assert(pipe.connection.state != .accepting);
+        assert(pipe.connection.state != .connecting);
 
         assert(pipe.status == .send);
         pipe.status = .idle;
@@ -268,20 +271,14 @@ const Connection = struct {
     connect_completion: IO.Completion = undefined,
     close_completion: IO.Completion = undefined,
 
-    fn on_accept(
+    fn accept_callback(
         connection: *Connection,
         _: *IO.Completion,
         result: IO.AcceptError!std.posix.socket_t,
     ) void {
         assert(connection.state == .accepting);
-        defer assert(connection.state == .connecting);
-
         assert(connection.origin_fd == null);
-        defer assert(connection.origin_fd != null);
-
         assert(connection.remote_fd == null);
-        defer assert(connection.remote_fd != null);
-
         assert(connection.remote_address != null);
 
         const fd = result catch |err| {
@@ -305,21 +302,21 @@ const Connection = struct {
             });
             return connection.try_close();
         };
-        connection.remote_fd = remote_fd;
 
+        connection.remote_fd = remote_fd;
         connection.state = .connecting;
 
         connection.io.connect(
             *Connection,
             connection,
-            Connection.on_connect,
+            Connection.connect_callback,
             &connection.connect_completion,
             connection.remote_fd.?,
             connection.remote_address.?,
         );
     }
 
-    fn on_connect(
+    fn connect_callback(
         connection: *Connection,
         _: *IO.Completion,
         result: IO.ConnectError!void,
@@ -357,41 +354,37 @@ const Connection = struct {
                 connection.connection_index,
             });
             connection.state = .closing;
-            std.posix.shutdown(connection.origin_fd.?, .both) catch |err| {
-                switch (err) {
-                    error.SocketNotConnected => {},
-                    else => log.warn("shutdown origin_fd ({d},{d}) failed: {}", .{
-                        connection.replica_index, connection.connection_index, err,
-                    }),
-                }
+            std.posix.shutdown(connection.origin_fd.?, .both) catch |err| switch (err) {
+                error.SocketNotConnected => {},
+                else => log.warn("shutdown origin_fd ({d},{d}) failed: {}", .{
+                    connection.replica_index, connection.connection_index, err,
+                }),
             };
-            std.posix.shutdown(connection.remote_fd.?, .both) catch |err| {
-                switch (err) {
-                    error.SocketNotConnected => {},
-                    else => log.warn("shutdown remote_fd ({d},{d}) failed: {}", .{
-                        connection.replica_index, connection.connection_index, err,
-                    }),
-                }
+            std.posix.shutdown(connection.remote_fd.?, .both) catch |err| switch (err) {
+                error.SocketNotConnected => {},
+                else => log.warn("shutdown remote_fd ({d},{d}) failed: {}", .{
+                    connection.replica_index, connection.connection_index, err,
+                }),
             };
         }
 
-        if (!has_inflight_operations) {
+        if (has_inflight_operations) {
+            // Network.tick() will keep calling try_close().
+            assert(connection.state == .closing);
+        } else {
             // Kick off the close sequence.
             connection.state = .closing_origin;
             connection.io.close(
                 *Connection,
                 connection,
-                on_close_origin,
+                close_origin_callback,
                 &connection.close_completion,
                 connection.origin_fd.?,
             );
-            return;
         }
-
-        assert(connection.state == .closing);
     }
 
-    fn on_close_origin(
+    fn close_origin_callback(
         connection: *Connection,
         _: *IO.Completion,
         result: IO.CloseError!void,
@@ -403,7 +396,7 @@ const Connection = struct {
         defer assert(connection.origin_fd == null);
 
         result catch |err| {
-            log.warn("on_close_origin ({d},{d}) error: {any}", .{
+            log.warn("close_origin_callback ({d},{d}) error: {any}", .{
                 connection.replica_index,
                 connection.connection_index,
                 err,
@@ -415,13 +408,13 @@ const Connection = struct {
         connection.io.close(
             *Connection,
             connection,
-            on_close_remote,
+            close_remote_callback,
             &connection.close_completion,
             connection.remote_fd.?,
         );
     }
 
-    fn on_close_remote(
+    fn close_remote_callback(
         connection: *Connection,
         _: *IO.Completion,
         result: IO.CloseError!void,
@@ -433,19 +426,22 @@ const Connection = struct {
         defer assert(connection.remote_fd == null);
 
         result catch |err| {
-            log.warn("on_close_remote ({d},{d}) error: {any}", .{
+            log.warn("close_remote_callback ({d},{d}) error: {any}", .{
                 connection.replica_index,
                 connection.connection_index,
                 err,
             });
         };
 
-        log.debug("on_close_remote ({d},{d}): marking connection as free", .{
+        log.debug("close_remote_callback ({d},{d}): marking connection as free", .{
             connection.replica_index,
             connection.connection_index,
         });
         connection.state = .free;
         connection.remote_fd = null;
+        connection.remote_address = null;
+        connection.origin_to_remote_pipe = .{ .io = connection.io, .connection = connection };
+        connection.remote_to_origin_pipe = .{ .io = connection.io, .connection = connection };
     }
 };
 
@@ -543,8 +539,10 @@ pub const Network = struct {
     }
 
     pub fn tick(network: *Network) void {
-        for (network.proxies) |*proxy| {
+        for (network.proxies, 0..) |*proxy, replica_index| {
             for (&proxy.connections) |*connection| {
+                assert(connection.replica_index == replica_index);
+
                 if (connection.state == .closing) {
                     connection.try_close();
                     continue;
@@ -555,6 +553,9 @@ pub const Network = struct {
                 if (connection.state == .free) {
                     assert(connection.origin_to_remote_pipe.status == .idle);
                     assert(connection.remote_to_origin_pipe.status == .idle);
+                    assert(connection.origin_fd == null);
+                    assert(connection.remote_fd == null);
+                    assert(connection.remote_address == null);
 
                     log.debug("accepting ({d},{d})", .{
                         connection.replica_index,
@@ -563,13 +564,11 @@ pub const Network = struct {
 
                     connection.state = .accepting;
                     connection.remote_address = proxy.remote_address;
-                    connection.origin_fd = null;
-                    connection.remote_fd = null;
 
                     network.io.accept(
                         *Connection,
                         connection,
-                        Connection.on_accept,
+                        Connection.accept_callback,
                         &connection.accept_completion,
                         proxy.accept_fd,
                     );

data/ext/tb_client/tigerbeetle/src/testing/vortex/supervisor.zig

@@ -296,15 +296,13 @@ const Supervisor = struct {
                 const too_slow_request = supervisor.workload.find_slow_request_since(start_ns);
 
                 if (no_finished_requests) {
-                    log.err("liveness check: no finished requests after {d} seconds", .{
+                    fatal(.liveness, "liveness check: no finished requests after {d} seconds", .{
                         constants.vortex.liveness_requirement_seconds,
                     });
-                    return error.TestFailed;
                 }
 
                 if (too_slow_request) |_| {
-                    log.err("liveness check: too slow request", .{});
-                    return error.TestFailed;
+                    fatal(.request_slow, "liveness check: too slow request", .{});
                 }
             }
 
@@ -335,7 +333,6 @@ const Supervisor = struct {
                     replica_pause,
                     replica_resume,
                     network_delay,
-                    network_lose,
                     network_corrupt,
                     network_heal,
                     quiesce,
@@ -348,7 +345,6 @@ const Supervisor = struct {
                     .replica_pause = if (running_replicas.len > 0) 3 else 0,
                     .replica_resume = if (paused_replicas.len > 0) 10 else 0,
                     .network_delay = if (supervisor.network.faults.delay == null) 3 else 0,
-                    .network_lose = if (supervisor.network.faults.lose == null) 3 else 0,
                     .network_corrupt = if (supervisor.network.faults.corrupt == null) 3 else 0,
                     .network_heal = if (!supervisor.network.faults.is_healed()) 10 else 0,
                     .quiesce = if (faulty_replica_count > 0 or
@@ -388,11 +384,6 @@ const Supervisor = struct {
                         };
                         log.info("injecting network delays: {any}", .{supervisor.network.faults});
                     },
-                    .network_lose => {
-                        supervisor.network.faults.lose =
-                            ratio(supervisor.prng.range_inclusive(u8, 1, 10), 100);
-                        log.info("injecting network loss: {any}", .{supervisor.network.faults});
-                    },
                     .network_corrupt => {
                         supervisor.network.faults.corrupt =
                             ratio(supervisor.prng.range_inclusive(u8, 1, 10), 100);
@@ -440,15 +431,18 @@ const Supervisor = struct {
                             // success or failure.
                             std.posix.exit(@intCast(128 + term.Signal));
                         } else {
-                            return error.TestFailed;
+                            fatal(.replica_exit_result, "replica exited with: {}", .{term});
                         }
                     }
                 }
             }
 
             if (supervisor.workload.process.wait_nonblocking()) |code| {
-                log.err("workload terminated by itself: code={}", .{code});
-                return error.TestFailed;
+                fatal(
+                    .workload_exit_early,
+                    "workload terminated by itself: code={}",
+                    .{code},
+                );
             }
         } else blk: {
             log.info("terminating workload due to max duration", .{});
@@ -460,14 +454,20 @@ const Supervisor = struct {
                 switch (signal) {
                     std.posix.SIG.KILL => log.info("workload terminated as requested", .{}),
                     else => {
-                        log.err("workload exited unexpectedly with signal {d}", .{signal});
-                        return error.TestFailed;
+                        fatal(
+                            .workload_exit_result,
+                            "workload exited unexpectedly with signal {d}",
+                            .{signal},
+                        );
                     },
                 }
             },
             else => {
-                log.err("unexpected workload result: {any}", .{workload_result});
-                return error.TestFailed;
+                fatal(
+                    .workload_exit_result,
+                    "unexpected workload result: {any}",
+                    .{workload_result},
+                );
             },
         }
     }
@@ -724,8 +724,7 @@ const Workload = struct {
         if (workload.process.state != .running) return;
 
         const count = result catch |err| {
-            log.err("couldn't read from workload stdout: {}", .{err});
-            return;
+            fatal(.workload_read_error, "couldn't read from workload stdout: {}", .{err});
         };
 
         workload.read_progress += count;
@@ -764,3 +763,23 @@ const Workload = struct {
         return null;
     }
 };
+
+const FatalReason = enum(u8) {
+    workload_exit_early = 10,
+    workload_exit_result = 11,
+    workload_read_error = 12,
+    replica_exit_result = 13,
+    liveness = 14,
+    request_slow = 15,
+
+    pub fn exit_status(reason: FatalReason) u8 {
+        return @intFromEnum(reason);
+    }
+};
+
+fn fatal(reason: FatalReason, comptime fmt: []const u8, args: anytype) noreturn {
+    log.err(fmt, args);
+    const status = reason.exit_status();
+    assert(status != 0);
+    std.process.exit(status);
+}

data/ext/tb_client/tigerbeetle/src/tidy.zig

@@ -1220,6 +1220,7 @@ const DeadFilesDetector = struct {
             "build_multiversion.zig",
             "build.zig",
             "dotnet_bindings.zig",
+            "fetch.zig",
             "file_checker.zig",
             "fuzz_tests.zig",
             "go_bindings.zig",
@@ -1232,10 +1233,10 @@ const DeadFilesDetector = struct {
             "node.zig",
             "page_writer.zig",
             "python_bindings.zig",
+            "rust_bindings.zig",
             "scripts.zig",
             "search_index_writer.zig",
             "service_worker_writer.zig",
-            "rust_bindings.zig",
             "single_page_writer.zig",
             "tb_client_header.zig",
             "unit_tests.zig",

data/ext/tb_client/tigerbeetle/src/tigerbeetle/main.zig

@@ -63,7 +63,7 @@ pub const std_options: std.Options = .{
 pub fn main() !void {
     if (builtin.os.tag == .windows) try vsr.multiversion.wait_for_parent_to_exit();
 
-    var arena_instance = std.heap.ArenaAllocator.init(std.heap.page_allocator);
+    var arena_instance = std.heap.ArenaAllocator.init(stdx.huge_page_allocator);
     defer arena_instance.deinit();
 
     // Arena is an implementation detail, all memory must be freed.

data/ext/tb_client/tigerbeetle/src/vsr/journal.zig

@@ -338,7 +338,7 @@ pub fn JournalType(comptime Replica: type, comptime Storage: type) type {
             ))[0..constants.journal_iops_write_max];
             errdefer allocator.free(write_headers_sectors);
 
-            log.debug("{}: slot_count={} size={} headers_size={} prepares_size={}", .{
+            log.info("{}: slot_count={} size={} headers_size={} prepares_size={}", .{
                 replica,
                 slot_count,
                 std.fmt.fmtIntSizeBin(write_ahead_log_zone_size),