tigerbeetle 0.0.34 → 0.0.37

data/ext/tb_client/tigerbeetle/src/stdx/time_units.zig

@@ -0,0 +1,275 @@
+const std = @import("std");
+const assert = std.debug.assert;
+const stdx = @import("stdx.zig");
+
+/// A moment in monotonic time not anchored to any particular epoch.
+///
+/// The absolute value of `ns` is meaningless, but it is possible to compute `Duration` between
+/// two `Instant`s sourced from the same clock.
+///
+/// See also `InstantUnix`.
+pub const Instant = struct {
+    ns: u64,
+
+    pub fn add(now: Instant, duration: Duration) Instant {
+        return .{ .ns = now.ns + duration.ns };
+    }
+
+    pub fn duration_since(now: Instant, earlier: Instant) Duration {
+        assert(now.ns >= earlier.ns);
+        const elapsed_ns = now.ns - earlier.ns;
+        return .{ .ns = elapsed_ns };
+    }
+};
+
+/// Non-negative time difference between two `Instant`s.
+pub const Duration = struct {
+    ns: u64,
+
+    pub fn ms(amount_ms: u64) Duration {
+        return .{ .ns = amount_ms * std.time.ns_per_ms };
+    }
+
+    pub fn seconds(amount_seconds: u64) Duration {
+        return .{ .ns = amount_seconds * std.time.ns_per_s };
+    }
+
+    pub fn minutes(amount_minutes: u64) Duration {
+        return .{ .ns = amount_minutes * std.time.ns_per_min };
+    }
+
+    // Duration in microseconds, μs, 1/1_000_000 of a second.
+    pub fn to_us(duration: Duration) u64 {
+        return @divFloor(duration.ns, std.time.ns_per_us);
+    }
+
+    // Duration in milliseconds, ms, 1/1_000 of a second.
+    pub fn to_ms(duration: Duration) u64 {
+        return @divFloor(duration.ns, std.time.ns_per_ms);
+    }
+
+    pub fn min(lhs: Duration, rhs: Duration) Duration {
+        return .{ .ns = @min(lhs.ns, rhs.ns) };
+    }
+
+    pub fn max(lhs: Duration, rhs: Duration) Duration {
+        return .{ .ns = @max(lhs.ns, rhs.ns) };
+    }
+
+    pub fn clamp(duration: Duration, clamp_min: Duration, clamp_max: Duration) Duration {
+        assert(clamp_min.ns <= clamp_max.ns);
+        if (duration.ns < clamp_min.ns) return clamp_min;
+        if (duration.ns > clamp_max.ns) return clamp_max;
+        return duration;
+    }
+
+    pub const sort = struct {
+        pub fn asc(ctx: void, lhs: Duration, rhs: Duration) bool {
+            return std.sort.asc(u64)(ctx, lhs.ns, rhs.ns);
+        }
+    };
+
+    // Human readable format like `1.123s`.
+    // NB: this is a lossy operation, durations are rounded to look nice.
+    pub fn format(
+        duration: Duration,
+        comptime fmt: []const u8,
+        options: std.fmt.FormatOptions,
+        writer: anytype,
+    ) !void {
+        try std.fmt.fmtDuration(duration.ns).format(fmt, options, writer);
+    }
+
+    pub fn parse_flag_value(
+        string: []const u8,
+        static_diagnostic: *?[]const u8,
+    ) error{InvalidFlagValue}!Duration {
+        assert(string.len > 0);
+        var string_remaining = string;
+
+        var result: Duration = .{ .ns = 0 };
+        while (string_remaining.len > 0) {
+            string_remaining, const component =
+                try parse_flag_value_component(string_remaining, static_diagnostic);
+            result.ns +|= component.ns;
+        }
+
+        if (result.ns >= 1_000 * std.time.ns_per_day) {
+            static_diagnostic.* = "duration too large:";
+            return error.InvalidFlagValue;
+        }
+        return result;
+    }
+
+    fn parse_flag_value_component(
+        string: []const u8,
+        static_diagnostic: *?[]const u8,
+    ) error{InvalidFlagValue}!struct { []const u8, Duration } {
+        const split_index = for (string, 0..) |c, index| {
+            if (std.ascii.isDigit(c)) {
+                // Numeric part continues.
+            } else break index;
+        } else {
+            static_diagnostic.* = "missing unit; must be one of: d/h/m/s/ms/us/ns:";
+            return error.InvalidFlagValue;
+        };
+
+        if (split_index == 0) {
+            static_diagnostic.* = "missing value:";
+            return error.InvalidFlagValue;
+        }
+
+        const string_amount = string[0..split_index];
+        const string_remaining = string[split_index..];
+        assert(string_amount.len > 0);
+        assert(string_remaining.len > 0);
+
+        const amount = std.fmt.parseInt(u64, string_amount, 10) catch |err| switch (err) {
+            error.Overflow => {
+                static_diagnostic.* = "integer overflow:";
+                return error.InvalidFlagValue;
+            },
+            error.InvalidCharacter => unreachable,
+        };
+
+        const Unit = enum(u64) {
+            ns = 1,
+            us = std.time.ns_per_us,
+            ms = std.time.ns_per_ms,
+            s = std.time.ns_per_s,
+            m = std.time.ns_per_min,
+            h = std.time.ns_per_hour,
+            d = std.time.ns_per_day,
+        };
+
+        inline for (comptime std.enums.values(Unit)) |unit| {
+            if (stdx.cut_prefix(string_remaining, @tagName(unit))) |suffix| {
+                return .{ suffix, .{ .ns = amount *| @intFromEnum(unit) } };
+            }
+        } else {
+            static_diagnostic.* = "unknown unit; must be one of: d/h/m/s/ms/us/ns:";
+            return error.InvalidFlagValue;
+        }
+    }
+};
+
+test "Instant/Duration" {
+    const instant_1: Instant = .{ .ns = 100 * std.time.ns_per_day };
+    const instant_2: Instant = .{ .ns = 100 * std.time.ns_per_day + std.time.ns_per_s };
+    assert(instant_1.duration_since(instant_1).ns == 0);
+    assert(instant_2.duration_since(instant_1).ns == std.time.ns_per_s);
+
+    const duration = instant_2.duration_since(instant_1);
+    assert(duration.ns == 1_000_000_000);
+    assert(duration.to_us() == 1_000_000);
+    assert(duration.to_ms() == 1_000);
+
+    assert(Duration.ms(1).ns == std.time.ns_per_ms);
+    assert(Duration.seconds(1).ns == std.time.ns_per_s);
+    assert(Duration.minutes(1).ns == std.time.ns_per_min);
+}
+
+test "Duration.parse_flag_value" {
+    try stdx.parse_flag_value_fuzz(Duration, Duration.parse_flag_value, .{
+        .ok = &.{
+            .{ "1h", .{ .ns = std.time.ns_per_hour } },
+            .{ "1m", .{ .ns = std.time.ns_per_min } },
+            .{ "1h2m", .{ .ns = std.time.ns_per_hour + 2 * std.time.ns_per_min } },
+            .{ "1ms2us3ns", .{ .ns = std.time.ns_per_ms + 2 * std.time.ns_per_us + 3 } },
+        },
+        .err = &.{
+            .{ "h", "missing value" },
+            .{ "1", "missing unit" },
+            .{ "h1", "missing value" },
+            .{ "1H", "unknown unit; must be one of: d/h/m/s/ms/us/ns" },
+            .{ "1h2x", "unknown unit" },
+            .{ "1_0h", "unknown unit" },
+            .{ "1h 2m", "missing value" },
+            .{ "18446744073709551616ns", "integer overflow" },
+            .{ "1844674407370955161s", "duration too large" },
+        },
+    });
+}
+
+/// A moment in non-monotonic Unix time.
+/// Timestamp is relative to epoch 1970-01-1.
+///
+/// See also `Instant`.
+pub const InstantUnix = struct {
+    ns: u64,
+
+    pub fn now() InstantUnix {
+        const timestamp_ns = std.time.nanoTimestamp();
+        assert(timestamp_ns > 0);
+        assert(timestamp_ns <= std.math.maxInt(u64));
+        return .{ .ns = @intCast(timestamp_ns) };
+    }
+
+    pub fn from_timestamp_s(timestamp_s: u64) InstantUnix {
+        return InstantUnix{ .ns = timestamp_s * std.time.ms_per_s * std.time.ns_per_ms };
+    }
+
+    pub fn date_time(instant: InstantUnix) struct {
+        year: u16,
+        month: u8,
+        day: u8,
+        hour: u8,
+        minute: u8,
+        second: u8,
+        millisecond: u16,
+    } {
+        const timestamp_ms = @divTrunc(instant.ns, std.time.ns_per_ms);
+        const epoch_seconds = std.time.epoch.EpochSeconds{ .secs = @divTrunc(timestamp_ms, 1000) };
+        const year_day = epoch_seconds.getEpochDay().calculateYearDay();
+        const month_day = year_day.calculateMonthDay();
+        const time = epoch_seconds.getDaySeconds();
+
+        return .{
+            .year = year_day.year,
+            .month = month_day.month.numeric(),
+            .day = month_day.day_index + 1,
+            .hour = time.getHoursIntoDay(),
+            .minute = time.getMinutesIntoHour(),
+            .second = time.getSecondsIntoMinute(),
+            .millisecond = @intCast(@mod(timestamp_ms, 1000)),
+        };
+    }
+
+    pub fn format(
+        instant: InstantUnix,
+        comptime fmt: []const u8,
+        options: std.fmt.FormatOptions,
+        writer: anytype,
+    ) !void {
+        _ = fmt;
+        _ = options;
+        const datetime = instant.date_time();
+        try writer.print("{d:0>4}-{d:0>2}-{d:0>2} {d:0>2}:{d:0>2}:{d:0>2}.{d:0>3}Z", .{
+            datetime.year,
+            datetime.month,
+            datetime.day,
+            datetime.hour,
+            datetime.minute,
+            datetime.second,
+            datetime.millisecond,
+        });
+    }
+
+    pub fn to_seconds(instant: InstantUnix) u64 {
+        return @divFloor(instant.ns, std.time.ns_per_s);
+    }
+};
+
+test "InstantUnix format" {
+    const instant_min = InstantUnix{ .ns = 0 };
+    var buffer: [24]u8 = undefined;
+    try std.testing.expectEqualStrings(
+        "1970-01-01 00:00:00.000Z",
+        try std.fmt.bufPrint(&buffer, "{}", .{instant_min}),
+    );
+    const instant_max = InstantUnix{ .ns = std.math.maxInt(u64) };
+    try std.testing.expectEqualStrings(
+        "2554-07-21 23:34:33.709Z",
+        try std.fmt.bufPrint(&buffer, "{}", .{instant_max}),
+    );
+}

data/ext/tb_client/tigerbeetle/src/stdx/unshare.zig

@@ -0,0 +1,295 @@
+//! Some tools for working with Linux `unshare` and namespaces.
+//!
+//! We use user, pid, and network namespaces for two purposes:
+//!
+//! - Processes namespaces enable all processes in the namespace
+//!   to be killed when the namespace's init process is.
+//! - Network namespaces allow us to create an isolated loopback network.
+//!
+//! This code uses the Linux `unshare` syscall to create new
+//! namespaces.
+//!
+//! The main tool here is `maybe_unshare_and_relaunch`, which provides
+//! a pattern for forking a new process that is an init process in
+//! its own process namespace.
+
+const std = @import("std");
+const builtin = @import("builtin");
+const linux = std.os.linux;
+const log = std.log.scoped(.unshare);
+const assert = std.debug.assert;
+
+// The external pid of the init process of the unshare pid namespace.
+// (Its pid within the namespace is always 1.)
+var child_pid: ?std.process.Child.Id = null;
+
+// On receiving SIGTERM, the parent must explicitly kill the pid namespace child process, since
+// it would otherwise keep running. Since the child is the init process, that automatically kills
+// all of its descendants too.
+const trap_action = std.posix.Sigaction{
+    .handler = .{ .handler = trap_handler },
+    .mask = std.posix.empty_sigset,
+    .flags = 0,
+};
+
+fn trap_handler(signal: i32) callconv(.c) void {
+    if (child_pid) |child| {
+        std.posix.kill(child, std.posix.SIG.KILL) catch |err| {
+            log.err("error killing sandboxed process: {}", .{err});
+        };
+    }
+    std.posix.exit(@intCast(@as(i32, 128) + signal));
+}
+
+/// Relaunch this process with new namespaces.
+///
+/// If the current process is already running with the namespaces configured as
+/// requested then this function does nothing. Otherwise it configures the
+/// namespaces and with them spawns a new process with the same arguments as
+/// the current process, waits for it, then exits the process directly (not
+/// returning from this function).
+///
+/// This should generally be called immediately from `main`.
+///
+/// If the `pid` option is provided then the spawned process will be the init
+/// process in a new pid namespace. When it is terminated all subprocesses
+/// transitively will also be terminated.
+///
+/// If the `network` option is provided then the spawned process and its
+/// subprocesses will have loopback network access only.
+pub fn maybe_unshare_and_relaunch(
+    gpa: std.mem.Allocator,
+    options: struct {
+        pid: bool,
+        network: bool,
+    },
+) !void {
+    comptime assert(builtin.os.tag == .linux);
+
+    if (std.os.linux.getpid() != 1) {
+        try linux_unshare(.{
+            .pid = options.pid,
+            .network = options.network,
+        });
+        if (options.network) {
+            try linux_ip_link_loopback();
+        }
+        if (options.pid) {
+            std.posix.sigaction(std.posix.SIG.TERM, &trap_action, null);
+            try fork_and_exit(gpa);
+        }
+    } else {
+        // We are within the pid namespace.
+        assert(options.pid);
+        assert(std.os.linux.getpid() == 1);
+        assert(child_pid == null);
+    }
+}
+
+/// Implementation of `unshare` somewhat like
+///
+/// ```
+/// unshare --user --net --pid
+/// ```
+///
+/// We're trying to accomplish two main things:
+///
+/// - creating a new pid namespace so that all subprocesses
+///   are automatically terminated when pid 1 (the forked
+///   vortex supervisor) is terminated.
+/// - creating a network sandbox
+///
+/// Note that on recent Ubuntu's this only works if AppArmour
+/// rules have been relaxed:
+///
+/// ```
+/// sudo sysctl -w kernel.apparmor_restrict_unprivileged_unconfined=0
+/// sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
+/// ```
+pub fn linux_unshare(options: struct {
+    pid: bool,
+    network: bool,
+}) !void {
+    comptime assert(builtin.os.tag == .linux);
+
+    // Create user namespace first.
+    const unshare_user_result = std.os.linux.unshare(linux.CLONE.NEWUSER);
+    const unshare_user_errno = std.os.linux.E.init(unshare_user_result);
+    if (unshare_user_errno != .SUCCESS) {
+        log.err("Failed to create user namespace: {}", .{unshare_user_errno});
+        return error.UnshareFailure;
+    }
+
+    // Create PID namespace.
+    if (options.pid) {
+        const unshare_pid_result = std.os.linux.unshare(linux.CLONE.NEWPID);
+        const unshare_pid_errno = std.os.linux.E.init(unshare_pid_result);
+        if (unshare_pid_errno != .SUCCESS) {
+            log.err("Failed to create pid namespace: {}", .{unshare_pid_errno});
+            return error.UnshareFailure;
+        }
+    }
+
+    // Create network namespace.
+    if (options.network) {
+        const unshare_net_result = std.os.linux.unshare(linux.CLONE.NEWNET);
+        const unshare_net_errno = std.os.linux.E.init(unshare_net_result);
+        if (unshare_net_errno != .SUCCESS) {
+            log.err("Failed to create net namespace: {}", .{unshare_net_errno});
+            return error.UnshareFailure;
+        }
+    }
+}
+
+/// Implementation of `ip link` equivalent to
+///
+/// ```
+/// ip link set up dev lo
+/// ```
+///
+/// This brings up the loopback device so that networking
+/// over 127.0.0.1 works.
+pub fn linux_ip_link_loopback() !void {
+    comptime assert(builtin.os.tag == .linux);
+
+    // Open a netlink socket with the NETLINK.ROUTE protocol.
+    const sock = std.posix.socket(
+        linux.AF.NETLINK,
+        std.posix.SOCK.RAW,
+        linux.NETLINK.ROUTE,
+    ) catch |err| {
+        log.err("failed to create netlink socket: {}", .{err});
+        return error.IpLink;
+    };
+    defer std.posix.close(sock);
+
+    const addr = linux.sockaddr.nl{
+        .family = linux.AF.NETLINK,
+        .pid = 0,
+        .groups = 0,
+    };
+    std.posix.bind(sock, @ptrCast(&addr), @sizeOf(@TypeOf(addr))) catch |err| {
+        log.err("failed to bind netlink socket: {}", .{err});
+        return error.IpLink;
+    };
+
+    // Netlink definitions.
+    const nlmsghdr = linux.nlmsghdr;
+    const ifinfomsg = linux.ifinfomsg;
+
+    const nlmsgerr = extern struct {
+        @"error": c_int,
+        msg: nlmsghdr,
+    };
+
+    const IFF_UP = 0x1;
+
+    // Our message to the kernel - header plus interface info.
+    const Message = extern struct {
+        hdr: nlmsghdr,
+        ifi: ifinfomsg,
+
+        comptime {
+            assert(@sizeOf(@This()) == @sizeOf(nlmsghdr) + @sizeOf(ifinfomsg));
+        }
+    };
+
+    // Kernel's message to us - header plus error info.
+    const Response = extern struct {
+        hdr: nlmsghdr,
+        err: nlmsgerr,
+
+        comptime {
+            assert(@sizeOf(@This()) == @sizeOf(nlmsghdr) + @sizeOf(nlmsgerr));
+        }
+    };
+
+    var msg: Message = .{
+        .hdr = .{
+            .len = @sizeOf(nlmsghdr) + @sizeOf(ifinfomsg),
+            .type = .RTM_NEWLINK,
+            // ACK says to always send a response, even on success.
+            .flags = linux.NLM_F_REQUEST | linux.NLM_F_ACK,
+            .seq = 0,
+            .pid = 0,
+        },
+        .ifi = .{
+            .family = linux.AF.UNSPEC,
+            .type = 0,
+            // Seems to be the loopback device, not sure how
+            // to find this value the correct way.
+            .index = 1,
+            .flags = IFF_UP,
+            // man pages say use this value.
+            .change = 0xFFFFFFFF,
+        },
+    };
+
+    const msg_buf = std.mem.asBytes(&msg);
+    const sent_len = std.posix.sendto(sock, msg_buf, 0, null, 0) catch |err| {
+        log.err("failed to send netlink message: {}", .{err});
+        return error.IpLink;
+    };
+    assert(sent_len == msg.hdr.len);
+
+    var ack: Response = undefined;
+    const ack_buf = std.mem.asBytes(&ack);
+    const ack_len = std.posix.recv(sock, ack_buf, 0) catch |err| {
+        log.err("failed to receive netlink ack: {}", .{err});
+        return error.IpLink;
+    };
+
+    assert(ack_len == @sizeOf(Response));
+    assert(ack.hdr.type == .ERROR);
+    assert(ack.err.msg.pid == msg.hdr.pid);
+
+    if (ack.err.@"error" != 0) {
+        log.err("netlink operation failed with errno: {}", .{-ack.err.@"error"});
+        return error.IpLink;
+    }
+}
+
+fn fork_and_exit(gpa: std.mem.Allocator) !void {
+    const args_ours = std.os.argv;
+
+    // We get a fresh path to the exe instead of using the original
+    // first argument so that the exe path will be correct even if
+    // this process's cwd has changed relative to the original exe.
+    var exe_path_buffer: [std.fs.max_path_bytes]u8 = undefined;
+    const exe_path = try std.fs.selfExePath(&exe_path_buffer);
+
+    const args_new = try gpa.alloc([]const u8, args_ours.len);
+    defer gpa.free(args_new);
+
+    args_new[0] = exe_path;
+
+    for (1..args_ours.len) |arg_index| {
+        args_new[arg_index] = std.mem.span(args_ours[arg_index]);
+    }
+
+    var child = std.process.Child.init(args_new, gpa);
+    child.stdin_behavior = .Inherit;
+    child.stdout_behavior = .Inherit;
+    child.stderr_behavior = .Inherit;
+
+    try child.spawn();
+
+    // Set the global pid so that we can kill it if we receive a SIGTERM.
+    assert(child_pid == null);
+    child_pid = child.id;
+
+    const result = try child.wait();
+    switch (result) {
+        .Exited => |code| {
+            std.process.exit(code);
+        },
+        .Signal => |signal| {
+            log.info("sandboxed subprocesses exited with signal {}", .{signal});
+            std.process.exit(1);
+        },
+        else => {
+            log.err("sandboxed subprocesses exited abnormally", .{});
+            std.process.exit(2);
+        },
+    }
+}