tigerbeetle 0.0.34 → 0.0.37

data/ext/tb_client/tigerbeetle/src/copyhound.zig

@@ -0,0 +1,202 @@
+//! Analyze LLVM IR to find:
+//! - large memcpy calls
+//! - functions with many copies due to monomorphisation and big total size
+//!
+//! To get a file with IR, use `-femit-llvm-ir` cli argument for `zig build-exe` or
+//!
+//!     $ zig build -Drelease -Demit-llvm-ir
+//!
+//! Pass the resulting .ll file to copyhound on stdin.
+//!
+//! ## Needless memcpy
+//!
+//! Run:
+//!
+//!     $ zig run -OReleaseSafe src/copyhound.zig -- memcpy --bytes 128 < tigerbeetle.ll \
+//!        | sort -n -k 2
+//!
+//! This only detects memory copies with comptime-know size (eg, when you copy a `T`, rather than a
+//! `[]T`).
+//!
+//! ## Code size
+//!
+//! Run:
+//!
+//!     $ zig run -OReleaseSafe src/copyhound.zig -- funcsize < tigerbeetle.ll \
+//!        | awk '{a[$1] += $2; b[$1] += 1} END {for (i in a) print i, b[i], a[i]}' \
+//!        | sort -n -k 3
+//!
+//! This will print every function name (first column), number of times it was monomorphized (second
+//! column) and the total size of all monorphisations (third column).
+
+const std = @import("std");
+const stdx = @import("stdx");
+
+const MiB = stdx.MiB;
+
+const log = std.log;
+pub const std_options = .{
+    .log_level = .info,
+};
+
+const CLIArgs = union(enum) {
+    memcpy: struct { bytes: u32 },
+    funcsize,
+};
+
+pub fn main() !void {
+    var gpa = std.heap.GeneralPurposeAllocator(.{}){};
+    var arena = std.heap.ArenaAllocator.init(gpa.allocator());
+    defer arena.deinit();
+
+    const allocator = arena.allocator();
+
+    var args = try std.process.argsWithAllocator(allocator);
+
+    const cli_args = stdx.flags(&args, CLIArgs);
+
+    const line_buffer = try allocator.alloc(u8, MiB);
+    const func_buf = try allocator.alloc(u8, 4096);
+
+    const stdin = std.io.getStdIn();
+    var buf_reader = std.io.bufferedReader(stdin.reader());
+    var in_stream = buf_reader.reader();
+
+    const stdout = std.io.getStdOut();
+    var buf_writer = std.io.bufferedWriter(stdout.writer());
+    defer buf_writer.flush() catch {};
+
+    var out_stream = buf_writer.writer();
+
+    var current_function: ?[]const u8 = null;
+    var current_function_size: u32 = 0;
+    while (try in_stream.readUntilDelimiterOrEof(line_buffer, '\n')) |line| {
+        if (std.mem.startsWith(u8, line, "define ")) {
+            current_function = extract_function_name(line, func_buf) orelse {
+                log.err("can't parse define line={s}", .{line});
+                return error.BadDefine;
+            };
+            continue;
+        }
+
+        if (current_function) |function| {
+            if (std.mem.eql(u8, line, "}")) {
+                if (cli_args == .funcsize) {
+                    try out_stream.print("{s} {}\n", .{ function, current_function_size });
+                }
+                current_function = null;
+                current_function_size = 0;
+                continue;
+            }
+            current_function_size += 1;
+            if (stdx.cut(line, "@llvm.memcpy")) |cut| {
+                const size = extract_memcpy_size(cut.suffix) orelse {
+                    log.err("can't parse memcpy call line={s}", .{line});
+                    return error.BadMemcpy;
+                };
+                if (cli_args == .memcpy) {
+                    if (size > cli_args.memcpy.bytes) {
+                        try out_stream.print("{s} {}\n", .{ function, size });
+                    }
+                }
+            }
+        }
+    }
+}
+
+/// Demangles function name by removing all comptime arguments (which are always inside `()`).
+fn extract_function_name(define: []const u8, buf: []u8) ?[]const u8 {
+    if (!std.mem.endsWith(u8, define, "{")) return null;
+
+    _, const mangled_name = stdx.cut(define, "@") orelse return null;
+    var buf_count: usize = 0;
+    var level: u32 = 0;
+    for (mangled_name) |c| {
+        switch (c) {
+            '(' => level += 1,
+            ')' => level -= 1,
+            '"' => {},
+            else => {
+                if (level > 0) continue;
+                if (c == ' ') return buf[0..buf_count];
+                if (buf_count == buf.len) return null;
+                buf[buf_count] = c;
+                buf_count += 1;
+            },
+        }
+    } else return null;
+}
+
+test "extract_function_name" {
+    var buf: [1024]u8 = undefined;
+    const func_name = extract_function_name(
+        \\define internal fastcc i64 @".vsr.vsr.clock.ClockType(.vsr.time.Time).monotonic"
+    ++
+        \\(%.vsr.time.Time* %.0.1.val) unnamed_addr #1 !dbg !71485 {
+    , &buf).?;
+    try std.testing.expectEqualStrings(".vsr.vsr.clock.ClockType.monotonic", func_name);
+}
+
+/// Parses out the size argument of an memcpy call.
+fn extract_memcpy_size(memcpy_call: []const u8) ?u32 {
+    _, const call_args = stdx.cut(memcpy_call, "(") orelse return null;
+    var level: u32 = 0;
+    var arg_count: u32 = 0;
+
+    const args_after_size = for (call_args, 0..) |c, i| {
+        switch (c) {
+            '(' => level += 1,
+            ')' => level -= 1,
+            ',' => {
+                if (level > 0) continue;
+                arg_count += 1;
+                if (!std.mem.startsWith(u8, call_args[i..], ", ")) return null;
+                if (arg_count == 2) break call_args[i + 2 ..];
+            },
+            else => {},
+        }
+    } else return null;
+
+    const size_arg, _ = stdx.cut(args_after_size, ",") orelse return null;
+
+    _, const size_value = stdx.cut(size_arg, " ") orelse return null;
+
+    // Runtime-known memcpy size, assume that's OK.
+    if (std.mem.startsWith(u8, size_value, "%")) return 0;
+
+    return std.fmt.parseInt(u32, size_value, 10) catch null;
+}
+
+test "extract_memcpy_size" {
+    const T = struct {
+        fn check(
+            line: []const u8,
+            want: ?u32,
+        ) !void {
+            const got = extract_memcpy_size(line);
+            try std.testing.expectEqual(want, got);
+        }
+    };
+
+    // One argument is a nested expression with a function call.
+    try T.check(
+        "  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %0, i8* align 8 bitcast(" ++
+            "{ void (i32, %std.os.linux.siginfo_t*, i8*)*," ++
+            " [32 x i32], <{ i32, [4 x i8] }>, void ()* }*" ++
+            " @8 to i8*), i64 152, i1 false)",
+        152,
+    );
+
+    // The argument is `%6` --- a runtime value.
+    try T.check(
+        \\   call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 1 %8, i8* align 1 %4, i64 %6, i1 false)
+    , 0);
+}
+
+/// Format and print an error message followed by the usage string to stderr,
+/// then exit with an exit code of 1.
+pub fn fatal(comptime fmt_string: []const u8, args: anytype) noreturn {
+    const stderr = std.io.getStdErr().writer();
+    stderr.print("error: " ++ fmt_string ++ "\n", args) catch {};
+    std.posix.exit(1);
+}

data/ext/tb_client/tigerbeetle/src/counting_allocator.zig

@@ -0,0 +1,72 @@
+const std = @import("std");
+const Alignment = std.mem.Alignment;
+
+const CountingAllocator = @This();
+
+parent_allocator: std.mem.Allocator,
+alloc_size: u64 = 0,
+free_size: u64 = 0,
+
+pub fn init(parent_allocator: std.mem.Allocator) CountingAllocator {
+    return .{ .parent_allocator = parent_allocator };
+}
+
+pub fn deinit(self: *CountingAllocator) void {
+    self.* = undefined;
+}
+
+pub fn allocator(self: *CountingAllocator) std.mem.Allocator {
+    return .{
+        .ptr = self,
+        .vtable = &.{
+            .alloc = alloc,
+            .resize = resize,
+            .remap = remap,
+            .free = free,
+        },
+    };
+}
+
+pub fn live_size(self: *CountingAllocator) u64 {
+    return self.alloc_size - self.free_size;
+}
+
+fn alloc(ctx: *anyopaque, len: usize, ptr_align: Alignment, ret_addr: usize) ?[*]u8 {
+    const self: *CountingAllocator = @ptrCast(@alignCast(ctx));
+    self.alloc_size += len;
+    return self.parent_allocator.rawAlloc(len, ptr_align, ret_addr);
+}
+
+fn resize(ctx: *anyopaque, buf: []u8, buf_align: Alignment, new_len: usize, ret_addr: usize) bool {
+    const self: *CountingAllocator = @ptrCast(@alignCast(ctx));
+
+    if (self.parent_allocator.rawResize(buf, buf_align, new_len, ret_addr)) {
+        if (new_len > buf.len) {
+            self.alloc_size += new_len - buf.len;
+        } else {
+            self.free_size += buf.len - new_len;
+        }
+        return true;
+    } else {
+        return false;
+    }
+}
+
+fn remap(ctx: *anyopaque, buf: []u8, buf_align: Alignment, new_len: usize, ret_addr: usize) ?[*]u8 {
+    const self: *CountingAllocator = @ptrCast(@alignCast(ctx));
+    if (self.parent_allocator.rawRemap(buf, buf_align, new_len, ret_addr)) |remapped| {
+        if (new_len > buf.len) {
+            self.alloc_size += new_len - buf.len;
+        } else {
+            self.free_size += buf.len - new_len;
+        }
+        return remapped;
+    }
+    return null;
+}
+
+fn free(ctx: *anyopaque, buf: []u8, buf_align: Alignment, ret_addr: usize) void {
+    const self: *CountingAllocator = @ptrCast(@alignCast(ctx));
+    self.free_size += buf.len;
+    return self.parent_allocator.rawFree(buf, buf_align, ret_addr);
+}

data/ext/tb_client/tigerbeetle/src/direction.zig

@@ -0,0 +1,11 @@
+pub const Direction = enum(u1) {
+    ascending = 0,
+    descending = 1,
+
+    pub fn reverse(d: Direction) Direction {
+        return switch (d) {
+            .ascending => .descending,
+            .descending => .ascending,
+        };
+    }
+};

data/ext/tb_client/tigerbeetle/src/docs_website/build.zig

@@ -0,0 +1,158 @@
+const std = @import("std");
+const Website = @import("src/website.zig").Website;
+const docs = @import("src/docs.zig");
+const redirects = @import("src/redirects.zig");
+
+pub const exclude_extensions: []const []const u8 = &.{
+    ".DS_Store",
+};
+
+pub fn build(b: *std.Build) !void {
+    const url_prefix: []const u8 = b.option(
+        []const u8,
+        "url_prefix",
+        "Prefix links with this string",
+    ) orelse "";
+
+    const git_commit = b.option(
+        []const u8,
+        "git-commit",
+        "The git commit revision of the source code.",
+    ) orelse std.mem.trimRight(u8, b.run(&.{ "git", "rev-parse", "--verify", "HEAD" }), "\n");
+
+    const pandoc_bin = get_pandoc_bin(b) orelse return;
+    const vale_bin = get_vale_bin(b) orelse return;
+
+    const check_spelling = std.Build.Step.Run.create(b, "run vale");
+    hide_stdout(check_spelling);
+    check_spelling.addFileArg(vale_bin);
+    const md_files = b.run(&.{ "git", "ls-files", "../../**/*.md" });
+    var md_files_iter = std.mem.tokenizeScalar(u8, md_files, '\n');
+    while (md_files_iter.next()) |md_file| {
+        check_spelling.addFileArg(b.path(md_file));
+    }
+
+    const content = b.addWriteFiles();
+    { //TODO(zig): https://github.com/ziglang/zig/issues/20571
+        var dir = try b.build_root.handle.openDir("assets", .{ .iterate = true });
+        defer dir.close();
+
+        var walker = try dir.walk(b.allocator);
+        defer walker.deinit();
+
+        while (try walker.next()) |entry| {
+            if (entry.kind == .file) {
+                if (std.mem.eql(u8, entry.basename, ".DS_Store")) continue;
+                const source = b.path("assets").path(b, entry.path);
+                _ = content.addCopyFile(source, entry.path);
+            }
+        }
+    }
+
+    content.step.dependOn(&check_spelling.step);
+
+    const website = Website.init(b, url_prefix, pandoc_bin);
+    try docs.build(b, content, website);
+    try redirects.build(b, content, website);
+
+    const clean_zigout_step = b.addRemoveDirTree(b.path("zig-out"));
+
+    const install_content_step = b.addInstallDirectory(.{
+        .source_dir = content.getDirectory(),
+        .install_dir = .prefix,
+        .install_subdir = ".",
+    });
+
+    install_content_step.step.dependOn(&clean_zigout_step.step);
+
+    const service_worker_writer = b.addRunArtifact(b.addExecutable(.{
+        .name = "service_worker_writer",
+        .root_source_file = b.path("src/service_worker_writer.zig"),
+        .target = b.graph.host,
+    }));
+    service_worker_writer.addArgs(&.{ url_prefix, git_commit });
+    service_worker_writer.addDirectoryArg(content.getDirectory());
+
+    const service_worker = service_worker_writer.captureStdOut();
+
+    const file_checker = b.addRunArtifact(b.addExecutable(.{
+        .name = "file_checker",
+        .root_source_file = b.path("src/file_checker.zig"),
+        .target = b.graph.host,
+    }));
+    file_checker.addArg("zig-out");
+
+    file_checker.step.dependOn(&install_content_step.step);
+    file_checker.step.dependOn(&b.addInstallFile(service_worker, "service-worker.js").step);
+
+    b.getInstallStep().dependOn(&file_checker.step);
+}
+
+fn get_pandoc_bin(b: *std.Build) ?std.Build.LazyPath {
+    const host = b.graph.host.result;
+    const name = switch (host.os.tag) {
+        .linux => switch (host.cpu.arch) {
+            .x86_64 => "pandoc_linux_amd64",
+            else => @panic("unsupported cpu arch"),
+        },
+        .macos => switch (host.cpu.arch) {
+            .aarch64 => "pandoc_macos_arm64",
+            else => @panic("unsupported cpu arch"),
+        },
+        else => @panic("unsupported os"),
+    };
+    if (b.lazyDependency(name, .{})) |dep| {
+        return dep.path("bin/pandoc");
+    } else {
+        return null;
+    }
+}
+
+fn get_vale_bin(b: *std.Build) ?std.Build.LazyPath {
+    const host = b.graph.host.result;
+    const name = switch (host.os.tag) {
+        .linux => switch (host.cpu.arch) {
+            .x86_64 => "vale_linux_amd64",
+            else => @panic("unsupported cpu arch"),
+        },
+        .macos => switch (host.cpu.arch) {
+            .aarch64 => "vale_macos_arm64",
+            else => @panic("unsupported cpu arch"),
+        },
+        else => @panic("unsupported os"),
+    };
+    if (b.lazyDependency(name, .{})) |dep| {
+        return dep.path("vale");
+    } else {
+        return null;
+    }
+}
+
+// Hide step's stdout unless it fails. Sadly, this requires overriding Build.Step.Run make function.
+fn hide_stdout(run: *std.Build.Step.Run) void {
+    _ = run.captureStdOut();
+
+    const override = struct {
+        var global_map: std.AutoHashMapUnmanaged(usize, std.Build.Step.MakeFn) = .{};
+
+        fn make(step: *std.Build.Step, options: std.Build.Step.MakeOptions) anyerror!void {
+            const original = global_map.get(@intFromPtr(step)).?;
+            original(step, options) catch |err| {
+                const run_step: *std.Build.Step.Run = @fieldParentPtr("step", step);
+                if (run_step.captured_stdout) |output| {
+                    const file = try std.fs.cwd().openFile(output.generated_file.getPath(), .{});
+                    defer file.close();
+
+                    const stdout = try file.readToEndAlloc(step.owner.allocator, 100 * 1024);
+                    std.debug.print("{s}\n", .{stdout});
+                }
+                return err;
+            };
+        }
+    };
+
+    const original = run.step.makeFn;
+    const b = run.step.owner;
+    override.global_map.put(b.allocator, @intFromPtr(&run.step), original) catch @panic("OOM");
+    run.step.makeFn = override.make;
+}

data/ext/tb_client/tigerbeetle/src/docs_website/src/content.zig

@@ -0,0 +1,156 @@
+//! Walk the `/docs` directory to build table of contents by parsing links to child pages from
+//! the READMEs.
+const std = @import("std");
+const Allocator = std.mem.Allocator;
+const assert = std.debug.assert;
+
+const log = std.log.scoped(.content);
+const cut_prefix = @import("./docs.zig").cut_prefix;
+const cut = @import("./docs.zig").cut;
+
+pub const Page = struct {
+    content: PageContent,
+    path: []const u8, // Always ends in .md
+    children: []Page,
+};
+
+// Parsed content of a single .md file.
+const PageContent = struct {
+    title: []const u8,
+    children: []Child,
+
+    const Child = struct {
+        title: []const u8,
+        path: []const u8,
+    };
+};
+
+pub fn load(arena: Allocator, base: std.fs.Dir, page_buffer: []u8) !Page {
+    return load_page(arena, base, "./", page_buffer);
+}
+
+fn load_page(arena: Allocator, base: std.fs.Dir, path: []const u8, page_buffer: []u8) !Page {
+    errdefer log.err("error while loading '{s}'", .{path});
+    const is_dir = std.mem.endsWith(u8, path, "/");
+    const is_client = std.mem.indexOf(u8, path, "src/clients") != null;
+
+    const file_path = if (is_dir) try std.fs.path.join(arena, &.{ path, "README.md" }) else path;
+    if (!std.mem.endsWith(u8, file_path, ".md")) {
+        return error.InvalidPath;
+    }
+
+    const text = try read_file(base, file_path, page_buffer);
+    const content = try parse_page_content(arena, text, .{
+        .parse_children = is_dir and !is_client,
+    });
+
+    var children: std.ArrayListUnmanaged(Page) = .{};
+    for (content.children) |child| {
+        assert(is_dir);
+        const child_path = if (std.mem.startsWith(u8, child.path, "/src/clients"))
+            try std.fs.path.join(arena, &.{
+                "..",
+                child.path,
+            })
+        else
+            try std.fs.path.join(arena, &.{
+                path,
+                cut_prefix(child.path, "./") orelse child.path,
+            });
+        const child_page = try load_page(arena, base, child_path, page_buffer);
+        try children.append(arena, child_page);
+    }
+
+    if (is_dir and !is_client) {
+        var dir = try base.openDir(path, .{ .iterate = true });
+        defer dir.close();
+
+        var dir_iterator = dir.iterate();
+        while (try dir_iterator.next()) |entry| {
+            if (std.mem.eql(u8, entry.name, "README.md")) continue;
+            if (std.mem.eql(u8, entry.name, "internals")) continue;
+            if (std.mem.eql(u8, entry.name, "TIGER_STYLE.md")) continue;
+            if (std.mem.eql(u8, entry.name, "ARCHITECTURE.md")) continue;
+            for (content.children) |child| {
+                const name = std.mem.trimRight(
+                    u8,
+                    cut_prefix(child.path, "./") orelse child.path,
+                    "/",
+                );
+                if (std.mem.eql(u8, name, entry.name)) break;
+            } else {
+                log.err("orphaned page: {s}{s}", .{ path, entry.name });
+                return error.OrphanedPage;
+            }
+        }
+    }
+
+    return .{
+        .content = content,
+        .path = file_path,
+        .children = children.items,
+    };
+}
+
+fn parse_page_content(arena: Allocator, text: []const u8, options: struct {
+    parse_children: bool,
+}) !PageContent {
+    var line_iterator = std.mem.tokenizeScalar(u8, text, '\n');
+    var title_line = line_iterator.next() orelse return error.TitleInvalid;
+    if (std.mem.startsWith(u8, title_line, "<!--")) {
+        // Clients' readmes start with auto-generated comment, skip over it.
+        title_line = line_iterator.next() orelse return error.TitleInvalid;
+    }
+
+    var title = cut_prefix(title_line, "# ") orelse return error.TitleInvalid;
+    title = std.mem.trim(u8, title, "`");
+    if (title.len < 3) return error.TitleInvalid;
+
+    var children: std.ArrayListUnmanaged(PageContent.Child) = .{};
+
+    if (options.parse_children) {
+        while (line_iterator.next()) |line| {
+            if (try parse_page_child(line)) |child| {
+                try children.append(arena, .{
+                    .title = try arena.dupe(u8, child.title),
+                    .path = try arena.dupe(u8, child.path),
+                });
+            }
+        }
+    }
+
+    return .{
+        .title = try arena.dupe(u8, title),
+        .children = children.items,
+    };
+}
+
+fn parse_page_child(line: []const u8) !?PageContent.Child {
+    errdefer log.err("error while parsing '{s}'", .{line});
+
+    var rest = cut_prefix(line, "- [") orelse return null;
+    const title, rest = cut(rest, "](") orelse return error.InvalidLink;
+    const path, _ = cut(rest, ")") orelse return error.InvalidLink;
+    if (!std.mem.startsWith(u8, path, "./")) {
+        if (std.mem.startsWith(u8, path, "/src/clients/")) {
+            // Special case.
+        } else {
+            log.err("ToC links must start with './'", .{});
+            return error.InvalidLink;
+        }
+    }
+    if (!std.mem.endsWith(u8, path, ".md") and !std.mem.endsWith(u8, path, "/")) {
+        log.err("ToC links must end with '/' or '.md'", .{});
+        return error.InvalidLink;
+    }
+    return .{
+        .title = std.mem.trim(u8, title, "`"),
+        .path = path,
+    };
+}
+
+fn read_file(dir: std.fs.Dir, path: []const u8, page_buffer: []u8) ![]const u8 {
+    const result = try dir.readFile(path, page_buffer);
+    if (result.len == page_buffer.len) return error.FileToLarge;
+    return result;
+}