tigerbeetle 0.0.36 → 0.0.37

data/ext/tb_client/tigerbeetle/src/testing/table.zig

@@ -0,0 +1,249 @@
+const std = @import("std");
+const assert = std.debug.assert;
+
+const stdx = @import("stdx");
+
+/// Parse a "table" of data with the specified schema.
+/// See test cases for example usage.
+pub fn parse(comptime Row: type, table_string: []const u8) stdx.BoundedArrayType(Row, 128) {
+    var rows = stdx.BoundedArrayType(Row, 128){};
+    var row_strings = std.mem.tokenizeScalar(u8, table_string, '\n');
+    while (row_strings.next()) |row_string| {
+        // Ignore blank line.
+        if (row_string.len == 0) continue;
+
+        var columns = std.mem.tokenizeScalar(u8, row_string, ' ');
+        const row = parse_data(Row, &columns);
+        rows.push(row);
+
+        // Ignore trailing line comment.
+        if (columns.next()) |last| assert(std.mem.eql(u8, last, "//"));
+    }
+    return rows;
+}
+
+fn parse_data(comptime Data: type, tokens: *std.mem.TokenIterator(u8, .scalar)) Data {
+    return switch (@typeInfo(Data)) {
+        .optional => |info| parse_data(info.child, tokens),
+        .@"enum" => field(Data, tokens.next().?),
+        .void => assert(tokens.next() == null),
+        .bool => {
+            const token = tokens.next().?;
+            inline for (.{ "0", "false", "F" }) |t| {
+                if (std.mem.eql(u8, token, t)) return false;
+            }
+            inline for (.{ "1", "true", "T" }) |t| {
+                if (std.mem.eql(u8, token, t)) return true;
+            }
+            std.debug.panic("Unknown boolean: {s}", .{token});
+        },
+        .int => |info| {
+            const max = std.math.maxInt(Data);
+            const token = tokens.next().?;
+            // If the first character is a letter ("a-zA-Z"), ignore it. (For example, "A1" → 1).
+            // This serves as a comment, to help visually disambiguate sequential integer columns.
+            const offset: usize = if (std.ascii.isAlphabetic(token[0])) 1 else 0;
+            // Negative unsigned values are computed relative to the maxInt.
+            if (info.signedness == .unsigned and token[offset] == '-') {
+                return max - (std.fmt.parseInt(Data, token[offset + 1 ..], 10) catch unreachable);
+            }
+            return std.fmt.parseInt(Data, token[offset..], 10) catch unreachable;
+        },
+        .@"struct" => {
+            var data: Data = undefined;
+            inline for (std.meta.fields(Data)) |value_field| {
+                const Field = value_field.type;
+                const value: Field = value: {
+                    if (comptime value_field.default_value_ptr) |ptr| {
+                        if (eat(tokens, "_")) {
+                            const value_ptr: *const Field = @ptrCast(@alignCast(ptr));
+                            break :value value_ptr.*;
+                        }
+                    }
+
+                    break :value parse_data(Field, tokens);
+                };
+
+                @field(data, value_field.name) = value;
+            }
+            return data;
+        },
+        .array => |info| {
+            var values: Data = undefined;
+            for (values[0..]) |*value| {
+                value.* = parse_data(info.child, tokens);
+            }
+            return values;
+        },
+        .@"union" => |info| {
+            const variant_string = tokens.next().?;
+            inline for (info.fields) |variant_field| {
+                if (std.mem.eql(u8, variant_field.name, variant_string)) {
+                    return @unionInit(
+                        Data,
+                        variant_field.name,
+                        parse_data(variant_field.type, tokens),
+                    );
+                }
+            }
+            std.debug.panic("Unknown union variant: {s}", .{variant_string});
+        },
+        else => @compileError("Unimplemented column type: " ++ @typeName(Data)),
+    };
+}
+
+fn eat(tokens: *std.mem.TokenIterator(u8, .scalar), token: []const u8) bool {
+    const index_before = tokens.index;
+    if (std.mem.eql(u8, tokens.next().?, token)) return true;
+    tokens.index = index_before;
+    return false;
+}
+
+/// TODO This function is a workaround for a comptime bug:
+///   error: unable to evaluate constant expression
+///   .@"enum" => @field(Column, column_string),
+fn field(comptime Enum: type, name: []const u8) Enum {
+    inline for (std.meta.fields(Enum)) |variant| {
+        if (std.mem.eql(u8, variant.name, name)) {
+            return @field(Enum, variant.name);
+        }
+    }
+    std.debug.panic("Unknown field name={s} for type={}", .{ name, Enum });
+}
+
+fn test_parse(
+    comptime Row: type,
+    comptime rows_expect: []const Row,
+    comptime string: []const u8,
+) !void {
+    const rows_actual = parse(Row, string).const_slice();
+    try std.testing.expectEqual(rows_expect.len, rows_actual.len);
+
+    for (rows_expect, 0..) |row, i| {
+        try std.testing.expectEqual(row, rows_actual[i]);
+    }
+}
+
+test "comment" {
+    try test_parse(struct {
+        a: u8,
+    }, &.{
+        .{ .a = 1 },
+    },
+        \\
+        \\ 1 // Comment
+        \\
+    );
+}
+
+test "enum" {
+    try test_parse(enum { a, b, c }, &.{ .c, .b, .a },
+        \\ c
+        \\ b
+        \\ a
+    );
+}
+
+test "bool" {
+    try test_parse(struct { i: bool }, &.{
+        .{ .i = false },
+        .{ .i = true },
+        .{ .i = false },
+        .{ .i = true },
+        .{ .i = false },
+        .{ .i = true },
+    },
+        \\ 0
+        \\ 1
+        \\ false
+        \\ true
+        \\ F
+        \\ T
+    );
+}
+
+test "int" {
+    try test_parse(struct { i: usize }, &.{
+        .{ .i = 1 },
+        .{ .i = 2 },
+        .{ .i = 3 },
+        .{ .i = 4 },
+        .{ .i = std.math.maxInt(usize) - 5 },
+        .{ .i = std.math.maxInt(usize) },
+    },
+        \\ 1
+        \\ 2
+        \\ A3
+        \\ a4
+        // For unsigned integers, `-n` is interpreted as `maxInt(Int) - n`.
+        \\ -5
+        \\ -0
+    );
+}
+
+test "struct" {
+    try test_parse(struct {
+        c1: enum { a, b, c, d },
+        c2: u8,
+        c3: u16 = 30,
+        c4: ?u32 = null,
+        c5: bool = false,
+    }, &.{
+        .{ .c1 = .a, .c2 = 1, .c3 = 10, .c4 = 1000, .c5 = true },
+        .{ .c1 = .b, .c2 = 2, .c3 = 20, .c4 = null, .c5 = true },
+        .{ .c1 = .c, .c2 = 3, .c3 = 30, .c4 = null, .c5 = false },
+        .{ .c1 = .d, .c2 = 4, .c3 = 30, .c4 = null, .c5 = false },
+    },
+        \\ a 1 10 1000 1
+        \\ b 2 20    _ T
+        \\ c 3  _    _ F
+        \\ d 4  _    _ _
+    );
+}
+
+test "struct (nested)" {
+    try test_parse(struct {
+        a: u32,
+        b: struct {
+            b1: u8,
+            b2: u8,
+        },
+        c: u32,
+    }, &.{
+        .{ .a = 1, .b = .{ .b1 = 2, .b2 = 3 }, .c = 4 },
+        .{ .a = 5, .b = .{ .b1 = 6, .b2 = 7 }, .c = 8 },
+    },
+        \\ 1 2 3 4
+        \\ 5 6 7 8
+    );
+}
+
+test "array" {
+    try test_parse(struct {
+        a: u32,
+        b: [2]u32,
+        c: u32,
+    }, &.{
+        .{ .a = 1, .b = .{ 2, 3 }, .c = 4 },
+        .{ .a = 5, .b = .{ 6, 7 }, .c = 8 },
+    },
+        \\ 1 2 3 4
+        \\ 5 6 7 8
+    );
+}
+
+test "union" {
+    try test_parse(union(enum) {
+        a: struct { b: u8, c: i8 },
+        d: u8,
+        e: void,
+    }, &.{
+        .{ .a = .{ .b = 1, .c = -2 } },
+        .{ .d = 3 },
+        .{ .e = {} },
+    },
+        \\a 1 -2
+        \\d 3
+        \\e
+    );
+}

data/ext/tb_client/tigerbeetle/src/testing/time.zig

@@ -0,0 +1,98 @@
+const std = @import("std");
+const stdx = @import("stdx");
+const Time = @import("../time.zig").Time;
+
+pub const OffsetType = enum {
+    linear,
+    periodic,
+    step,
+    non_ideal,
+};
+
+pub const TimeSim = struct {
+    /// The duration of a single tick in nanoseconds.
+    resolution: u64,
+
+    offset_type: OffsetType,
+
+    /// Co-efficients to scale the offset according to the `offset_type`.
+    /// Linear offset is described as A * x + B: A is the drift per tick and B the initial offset.
+    /// Periodic is described as A * sin(x * pi / B): A controls the amplitude and B the period in
+    /// terms of ticks.
+    /// Step function represents a discontinuous jump in the wall-clock time. B is the period in
+    /// which the jumps occur. A is the amplitude of the step.
+    /// Non-ideal is similar to periodic except the phase is adjusted using a random number taken
+    /// from a normal distribution with mean=0, stddev=10. Finally, a random offset (up to
+    /// offset_coefficient_C) is added to the result.
+    offset_coefficient_A: i64,
+    offset_coefficient_B: i64,
+    offset_coefficient_C: u32 = 0,
+
+    prng: stdx.PRNG = stdx.PRNG.from_seed(0),
+
+    /// The number of ticks elapsed since initialization.
+    ticks: u64 = 0,
+
+    /// The instant in time chosen as the origin of this time source.
+    epoch: i64 = 0,
+
+    pub fn time(self: *TimeSim) Time {
+        return .{
+            .context = self,
+            .vtable = &.{
+                .monotonic = monotonic,
+                .realtime = realtime,
+                .tick = tick,
+            },
+        };
+    }
+
+    fn monotonic(context: *anyopaque) u64 {
+        const self: *TimeSim = @ptrCast(@alignCast(context));
+
+        return self.ticks * self.resolution;
+    }
+
+    fn realtime(context: *anyopaque) i64 {
+        const self: *TimeSim = @ptrCast(@alignCast(context));
+
+        return self.epoch + @as(i64, @intCast(monotonic(context))) - self.offset(self.ticks);
+    }
+
+    pub fn offset(self: *TimeSim, ticks: u64) i64 {
+        switch (self.offset_type) {
+            .linear => {
+                const drift_per_tick = self.offset_coefficient_A;
+                return @as(i64, @intCast(ticks)) * drift_per_tick + @as(
+                    i64,
+                    @intCast(self.offset_coefficient_B),
+                );
+            },
+            .periodic => {
+                const unscaled = std.math.sin(@as(f64, @floatFromInt(ticks)) * 2 * std.math.pi /
+                    @as(f64, @floatFromInt(self.offset_coefficient_B)));
+                const scaled = @as(f64, @floatFromInt(self.offset_coefficient_A)) * unscaled;
+                return @as(i64, @intFromFloat(std.math.floor(scaled)));
+            },
+            .step => {
+                return if (ticks > self.offset_coefficient_B) self.offset_coefficient_A else 0;
+            },
+            .non_ideal => {
+                const phase: f64 = @as(f64, @floatFromInt(ticks)) * 2 * std.math.pi /
+                    (@as(f64, @floatFromInt(self.offset_coefficient_B)) +
+                        std.Random.init(&self.prng, stdx.PRNG.fill).floatNorm(f64) * 10);
+                const unscaled = std.math.sin(phase);
+                const scaled = @as(f64, @floatFromInt(self.offset_coefficient_A)) * unscaled;
+                const offset_random: i64 = -@as(i64, @intCast(self.offset_coefficient_C)) +
+                    @as(i64, @intCast(self.prng.int_inclusive(u64, 2 * self.offset_coefficient_C)));
+                return @as(i64, @intFromFloat(std.math.floor(scaled))) + offset_random;
+            },
+        }
+    }
+
+    fn tick(context: *anyopaque) void {
+        const self: *TimeSim = @ptrCast(@alignCast(context));
+
+        self.ticks += 1;
+    }
+};

data/ext/tb_client/tigerbeetle/src/testing/tmp_tigerbeetle.zig

@@ -0,0 +1,212 @@
+//! TmpTigerBeetle is an utility for integration tests, which spawns a single node TigerBeetle
+//! cluster in a temporary directory.
+
+const std = @import("std");
+const builtin = @import("builtin");
+const assert = std.debug.assert;
+
+const stdx = @import("stdx");
+const Shell = @import("../shell.zig");
+
+const MiB = stdx.MiB;
+
+const log = std.log.scoped(.tmptigerbeetle);
+
+const TmpTigerBeetle = @This();
+
+/// Path to the executable.
+tigerbeetle_exe: []const u8,
+/// Port the TigerBeetle instance is listening on.
+port: u16,
+/// For convenience, the same port pre-converted to string.
+port_str: []const u8,
+
+tmp_dir: std.testing.TmpDir,
+
+// A separate thread for reading process stderr without blocking it. The process must be terminated
+// before stopping the StreamReader.
+//
+// StreamReader echoes process' stderr on exit unless explicitly instructed otherwise.
+stderr_reader: *StreamReader,
+
+process: std.process.Child,
+
+pub fn init(
+    gpa: std.mem.Allocator,
+    options: struct {
+        development: bool,
+        prebuilt: ?[]const u8 = null,
+    },
+) !TmpTigerBeetle {
+    const shell = try Shell.create(gpa);
+    defer shell.destroy();
+
+    var from_source_path: ?[]const u8 = null;
+    defer if (from_source_path) |path| gpa.free(path);
+
+    if (options.prebuilt == null) {
+        const tigerbeetle_exe = comptime "tigerbeetle" ++ builtin.target.exeFileExt();
+
+        // If tigerbeetle binary does not exist yet, build it.
+        //
+        // TODO: just run `zig build run` unconditionally here, when that doesn't do spurious
+        // rebuilds.
+        _ = shell.project_root.statFile(tigerbeetle_exe) catch {
+            log.info("building TigerBeetle", .{});
+            try shell.exec_zig("build", .{});
+
+            _ = try shell.project_root.statFile(tigerbeetle_exe);
+        };
+
+        from_source_path = try shell.project_root.realpathAlloc(gpa, tigerbeetle_exe);
+    }
+
+    const tigerbeetle_exe: []const u8 = try gpa.dupe(
+        u8,
+        options.prebuilt orelse from_source_path.?,
+    );
+    errdefer gpa.free(tigerbeetle_exe);
+    assert(std.fs.path.isAbsolute(tigerbeetle_exe));
+
+    var tmp_dir = std.testing.tmpDir(.{});
+    errdefer tmp_dir.cleanup();
+
+    const tmp_dir_path = try tmp_dir.dir.realpathAlloc(gpa, ".");
+    defer gpa.free(tmp_dir_path);
+
+    const data_file: []const u8 = try std.fs.path.join(gpa, &.{ tmp_dir_path, "0_0.tigerbeetle" });
+    defer gpa.free(data_file);
+
+    try shell.exec(
+        "{tigerbeetle} format --cluster=0 --replica=0 --replica-count=1 {data_file}",
+        .{ .tigerbeetle = tigerbeetle_exe, .data_file = data_file },
+    );
+
+    var reader_maybe: ?*StreamReader = null;
+    // Pass `--addresses=0` to let the OS pick a port for us.
+    var process = try shell.spawn(
+        .{
+            .stdin_behavior = .Pipe,
+            .stdout_behavior = .Pipe,
+            .stderr_behavior = .Pipe,
+        },
+        "{tigerbeetle} start --development={development} --addresses=0 {data_file}",
+        .{
+            .tigerbeetle = tigerbeetle_exe,
+            .development = if (options.development) "true" else "false",
+            .data_file = data_file,
+        },
+    );
+
+    errdefer {
+        if (reader_maybe) |reader| {
+            reader.stop(gpa, &process); // Will log stderr.
+        } else {
+            _ = process.kill() catch unreachable;
+        }
+    }
+
+    reader_maybe = try StreamReader.start(gpa, process.stderr.?);
+
+    const port = port: {
+        var exit_status: ?std.process.Child.Term = null;
+        errdefer log.err(
+            "failed to read port number from tigerbeetle process: {?}",
+            .{exit_status},
+        );
+
+        var port_buf: [std.fmt.count("{}\n", .{std.math.maxInt(u16)})]u8 = undefined;
+        const port_buf_len = try process.stdout.?.readAll(&port_buf);
+        if (port_buf_len == 0) {
+            exit_status = try process.wait();
+            return error.NoPort;
+        }
+
+        break :port try std.fmt.parseInt(u16, port_buf[0 .. port_buf_len - 1], 10);
+    };
+
+    const port_str = try std.fmt.allocPrint(gpa, "{d}", .{port});
+    errdefer gpa.free(port_str);
+
+    return TmpTigerBeetle{
+        .tigerbeetle_exe = tigerbeetle_exe,
+        .port = port,
+        .port_str = port_str,
+        .tmp_dir = tmp_dir,
+        .stderr_reader = reader_maybe.?,
+        .process = process,
+    };
+}
+
+pub fn deinit(tb: *TmpTigerBeetle, gpa: std.mem.Allocator) void {
+    if (tb.stderr_reader.log_stderr.load(.seq_cst) == .on_early_exit) {
+        tb.stderr_reader.log_stderr.store(.no, .seq_cst);
+    }
+    assert(tb.process.term == null);
+    tb.stderr_reader.stop(gpa, &tb.process);
+    assert(tb.process.term != null);
+    gpa.free(tb.port_str);
+    tb.tmp_dir.cleanup();
+    gpa.free(tb.tigerbeetle_exe);
+}
+
+pub fn log_stderr(tb: *TmpTigerBeetle) void {
+    tb.stderr_reader.log_stderr.store(.yes, .seq_cst);
+}
+
+const StreamReader = struct {
+    const LogStderr = std.atomic.Value(enum(u8) { no, yes, on_early_exit });
+
+    log_stderr: LogStderr = LogStderr.init(.on_early_exit),
+    thread: std.Thread,
+    file: std.fs.File,
+
+    pub fn start(gpa: std.mem.Allocator, file: std.fs.File) !*StreamReader {
+        var result = try gpa.create(StreamReader);
+        errdefer gpa.destroy(result);
+
+        result.* = .{
+            .thread = undefined,
+            .file = file,
+        };
+
+        result.thread = try std.Thread.spawn(.{}, thread_main, .{result});
+        return result;
+    }
+
+    pub fn stop(self: *StreamReader, gpa: std.mem.Allocator, process: *std.process.Child) void {
+        // Shutdown sequence is tricky:
+        // 1. Terminate the process, but _don't_ close our side of the pipe.
+        // 2. Wait until the thread exits.
+        // 3. Close stderr file descriptor.
+        // TODO(Zig) https://github.com/ziglang/zig/issues/16820
+        if (builtin.os.tag == .windows) {
+            const exit_code = 1;
+            std.os.windows.TerminateProcess(process.id, exit_code) catch {};
+        } else {
+            std.posix.kill(process.id, std.posix.SIG.TERM) catch {};
+        }
+        assert(process.stderr != null);
+        self.thread.join();
+        _ = process.wait() catch unreachable;
+        assert(process.stderr == null);
+        gpa.destroy(self);
+    }
+
+    fn thread_main(reader: *StreamReader) void {
+        // NB: Zig allocators are not thread safe, so use mmap directly to hold process' stderr.
+        const allocator = std.heap.page_allocator;
+
+        var buffer = std.ArrayList(u8).init(allocator);
+        defer buffer.deinit();
+
+        // NB: don't use `readAllAlloc` to get partial output in case of errors.
+        reader.file.reader().readAllArrayList(&buffer, 100 * MiB) catch {};
+        switch (reader.log_stderr.load(.seq_cst)) {
+            .on_early_exit, .yes => {
+                log.err("tigerbeetle stderr:\n++++\n{s}\n++++", .{buffer.items});
+            },
+            .no => {},
+        }
+    }
+};

data/ext/tb_client/tigerbeetle/src/testing/vortex/constants.zig

@@ -0,0 +1,26 @@
+const std = @import("std");
+const constants = @This();
+
+pub const vsr = @import("../../constants.zig");
+
+pub const vortex = struct {
+    pub const cluster_id = 0;
+    pub const connections_count_max = @divFloor(
+        constants.vsr.clients_max,
+        constants.vsr.replicas_max,
+    );
+
+    // We allow the cluster to not make progress processing requests for this amount of time.
+    // After that it's considered a test failure.
+    pub const liveness_requirement_seconds = 120;
+    pub const liveness_requirement_micros = liveness_requirement_seconds * std.time.us_per_s;
+
+    pub const replica_ports_actual = brk: {
+        var ports: [constants.vsr.replicas_max]u16 = undefined;
+        var replica_num: u16 = 0;
+        while (replica_num < constants.vsr.replicas_max) : (replica_num += 1) {
+            ports[replica_num] = 4000 + replica_num;
+        }
+        break :brk ports;
+    };
+};