tigerbeetle 0.0.34 → 0.0.37

data/ext/tb_client/tigerbeetle/src/scripts/changelog.zig

@@ -0,0 +1,304 @@
+const std = @import("std");
+const assert = std.debug.assert;
+
+const stdx = @import("stdx");
+const Shell = @import("../shell.zig");
+
+const Release = @import("../multiversion.zig").Release;
+const ReleaseTriple = @import("../multiversion.zig").ReleaseTriple;
+
+const MiB = stdx.MiB;
+
+const log = std.log;
+
+const changelog_bytes_max = 10 * MiB;
+
+pub fn main(shell: *Shell, gpa: std.mem.Allocator) !void {
+    _ = gpa;
+
+    const date_time = stdx.InstantUnix.now().date_time();
+    const today = try shell.fmt(
+        "{:0>4}-{:0>2}-{:0>2}",
+        .{ date_time.year, date_time.month, date_time.day },
+    );
+
+    try shell.exec("git fetch origin --quiet", .{});
+    try shell.exec("git switch --create release-{today} origin/main", .{ .today = today });
+
+    const merges = try shell.exec_stdout(
+        \\git log --merges --first-parent origin/release..origin/main
+    , .{});
+
+    const changelog_current = try shell.project_root.readFileAlloc(
+        shell.arena.allocator(),
+        "./CHANGELOG.md",
+        changelog_bytes_max,
+    );
+
+    var changelog_new = std.ArrayList(u8).init(shell.arena.allocator());
+    try format_changelog(changelog_new.writer(), .{
+        .changelog_current = changelog_current,
+        .merges = merges,
+        .today = today,
+    });
+
+    try shell.project_root.writeFile(.{ .sub_path = "CHANGELOG.md", .data = changelog_new.items });
+
+    log.info("don't forget to update ./CHANGELOG.md", .{});
+}
+
+fn format_changelog(buffer: std.ArrayList(u8).Writer, options: struct {
+    changelog_current: []const u8,
+    merges: []const u8,
+    today: []const u8,
+}) !void {
+    if (std.mem.indexOf(u8, options.changelog_current, options.today) != null) {
+        return error.ChangelogAlreadyUpdated;
+    }
+
+    var it = ChangelogIterator.init(options.changelog_current);
+    const last_changelog_entry = it.next_changelog().?;
+
+    try buffer.print(
+        \\# Changelog
+        \\
+        \\Subscribe to the [tracking issue #2231](https://github.com/tigerbeetle/tigerbeetle/issues/2231)
+        \\to receive notifications about breaking changes!
+        \\
+        \\
+    , .{});
+
+    if (last_changelog_entry.release) |release| {
+        const release_next = Release.from(.{
+            .major = release.triple().major,
+            .minor = release.triple().minor,
+            .patch = release.triple().patch + 1,
+        });
+        try buffer.print("## TigerBeetle {}\n\n", .{release_next});
+    } else {
+        try buffer.print("## TigerBeetle (unreleased)\n\n", .{});
+    }
+    try buffer.print("Released: {s}\n\n", .{options.today});
+
+    var merges_left = options.merges;
+    for (0..128) |_| {
+        const merge = try format_changelog_cut_single_merge(&merges_left) orelse break;
+
+        try buffer.print(
+            \\- [#{d}](https://github.com/tigerbeetle/tigerbeetle/pull/{d})
+            \\
+            \\  {s}
+            \\
+            \\
+        , .{ merge.pr, merge.pr, merge.summary });
+    } else @panic("suspiciously many PRs merged");
+    assert(std.mem.indexOf(u8, merges_left, "commit") == null);
+
+    try buffer.print(
+        \\
+        \\### Safety And Performance
+        \\
+        \\-
+        \\
+        \\### Features
+        \\
+        \\-
+        \\
+        \\### Internals
+        \\
+        \\-
+        \\
+        \\### TigerTracks 🎧
+        \\
+        \\- []()
+        \\
+        \\
+    , .{});
+
+    try buffer.writeAll(it.all_entries);
+}
+
+fn format_changelog_cut_single_merge(merges_left: *[]const u8) !?struct {
+    pr: u16,
+    summary: []const u8,
+} {
+    errdefer {
+        log.err("failed to parse:\n{s}", .{merges_left.*});
+    }
+
+    // This is what we are parsing here:
+    //
+    //    commit 02650cd67da855609cc41196e0d6f639b870ccf5
+    //    Merge: b7c2fcda 4bb433ce
+    //    Author: protty <45520026+kprotty@users.noreply.github.com>
+    //    Date:   Fri Feb 9 18:37:04 2024 +0000
+    //
+    //    Merge pull request #1523 from tigerbeetle/king/client-uid
+    //
+    //    Client: add ULID helper functions
+
+    _, merges_left.* = stdx.cut(merges_left.*, "Merge pull request #") orelse return null;
+
+    const pr_string, merges_left.* = stdx.cut(merges_left.*, " from ") orelse
+        return error.ParseMergeLog;
+    const pr = try std.fmt.parseInt(u16, pr_string, 10);
+
+    _, merges_left.* = stdx.cut(merges_left.*, "\n    \n    ") orelse return error.ParseMergeLog;
+
+    const summary, merges_left.* = stdx.cut(merges_left.*, "\n") orelse return error.ParseMergeLog;
+
+    return .{ .pr = pr, .summary = summary };
+}
+
+pub const ChangelogIterator = struct {
+    const Entry = struct {
+        release: ?Release,
+        text_full: []const u8,
+        text_body: []const u8,
+    };
+
+    // Immutable suffix of the changelog, used to prepend a new entry in front.
+    all_entries: []const u8,
+
+    // Mutable suffix of what's yet to be iterated.
+    rest: []const u8,
+
+    release_previous_iteration: ?Release = null,
+
+    pub fn init(changelog: []const u8) ChangelogIterator {
+        var rest = stdx.cut_prefix(changelog, "# Changelog\n\n").?;
+        const start_index = std.mem.indexOf(u8, rest, "##").?;
+        assert(rest[start_index - 1] == '\n');
+        rest = rest[start_index..];
+        assert(std.mem.startsWith(u8, rest, "## TigerBeetle"));
+
+        return .{
+            .all_entries = rest,
+            .rest = rest,
+        };
+    }
+
+    pub fn next_changelog(it: *ChangelogIterator) ?Entry {
+        if (it.done()) return null;
+        assert(std.mem.startsWith(u8, it.rest, "## TigerBeetle"));
+        const entry_end_index = std.mem.indexOf(u8, it.rest[2..], "\n\n## ").? + 2;
+        const text_full = it.rest[0 .. entry_end_index + 1];
+        it.rest = it.rest[entry_end_index + 2 ..];
+        const entry = parse_entry(text_full);
+
+        if (it.release_previous_iteration != null and entry.release != null) {
+            // The changelog is ordered from newest to oldest, and that's how it's iterated. The
+            // current iteration's release is thus expected to be less than the previous iteration's
+            // release.
+            assert(Release.less_than({}, entry.release.?, it.release_previous_iteration.?));
+        }
+        if (entry.release != null) {
+            it.release_previous_iteration = entry.release;
+        }
+
+        return entry;
+    }
+
+    fn done(it: *const ChangelogIterator) bool {
+        // First old-style release.
+        return std.mem.startsWith(u8, it.rest, "## 2024-08-05");
+    }
+
+    fn parse_entry(text_full: []const u8) Entry {
+        assert(std.mem.startsWith(u8, text_full, "## TigerBeetle"));
+        assert(std.mem.endsWith(u8, text_full, "\n"));
+        assert(!std.mem.endsWith(u8, text_full, "\n\n"));
+
+        const first_line, var body = stdx.cut(text_full, "\n").?;
+        const release = if (std.mem.eql(u8, first_line, "## TigerBeetle (unreleased)"))
+            null
+        else
+            Release.parse(stdx.cut_prefix(first_line, "## TigerBeetle ").?) catch
+                @panic("invalid changelog");
+
+        body = stdx.cut_prefix(body, "\nReleased:").?;
+        _, body = stdx.cut(body, "\n").?;
+        return .{ .release = release, .text_full = text_full, .text_body = body };
+    }
+};
+
+test ChangelogIterator {
+    const changelog =
+        \\# Changelog
+        \\
+        \\Some preamble here
+        \\
+        \\## TigerBeetle 1.2.3
+        \\
+        \\Released: 2024-10-23
+        \\
+        \\This is the start of the changelog.
+        \\
+        \\### Features
+        \\
+        \\- a cool PR
+        \\
+        \\## TigerBeetle 1.2.2
+        \\
+        \\Released: 2024-10-16
+        \\
+        \\ The beginning.
+        \\
+        \\## 2024-08-05 (prehistory)
+        \\
+        \\
+    ;
+
+    var it = ChangelogIterator.init(changelog);
+
+    var entry = it.next_changelog().?;
+    try std.testing.expectEqual(entry.release.?.triple(), ReleaseTriple{
+        .major = 1,
+        .minor = 2,
+        .patch = 3,
+    });
+    try std.testing.expectEqualStrings(entry.text_full,
+        \\## TigerBeetle 1.2.3
+        \\
+        \\Released: 2024-10-23
+        \\
+        \\This is the start of the changelog.
+        \\
+        \\### Features
+        \\
+        \\- a cool PR
+        \\
+    );
+    try std.testing.expectEqualStrings(entry.text_body,
+        \\
+        \\This is the start of the changelog.
+        \\
+        \\### Features
+        \\
+        \\- a cool PR
+        \\
+    );
+
+    entry = it.next_changelog().?;
+    try std.testing.expectEqual(entry.release.?.triple(), ReleaseTriple{
+        .major = 1,
+        .minor = 2,
+        .patch = 2,
+    });
+
+    try std.testing.expectEqual(it.next_changelog(), null);
+}
+
+test "current changelog" {
+    const allocator = std.testing.allocator;
+
+    const changelog_text = try std.fs.cwd().readFileAlloc(
+        allocator,
+        "./CHANGELOG.md",
+        changelog_bytes_max,
+    );
+    defer allocator.free(changelog_text);
+
+    var it = ChangelogIterator.init(changelog_text);
+    while (it.next_changelog()) |_| {}
+}

data/ext/tb_client/tigerbeetle/src/scripts/ci.zig

@@ -0,0 +1,227 @@
+//! Various CI checks that go beyond `zig build test`. Notably, at the moment this script includes:
+//!
+//! - Testing all language clients.
+//! - Building and link-checking docs.
+
+const std = @import("std");
+const builtin = @import("builtin");
+const log = std.log;
+const assert = std.debug.assert;
+
+const stdx = @import("stdx");
+const Shell = @import("../shell.zig");
+
+const client_readmes = @import("./client_readmes.zig");
+
+pub const Language = std.meta.FieldEnum(@TypeOf(LanguageCI));
+const LanguageCI = .{
+    .dotnet = @import("../clients/dotnet/ci.zig"),
+    .go = @import("../clients/go/ci.zig"),
+    .rust = @import("../clients/rust/ci.zig"),
+    .java = @import("../clients/java/ci.zig"),
+    .node = @import("../clients/node/ci.zig"),
+    .python = @import("../clients/python/ci.zig"),
+};
+
+const LanguageCIVortex = .{
+    .rust = @import("../testing/vortex/rust_driver/ci.zig"),
+    .java = @import("../testing/vortex/java_driver/ci.zig"),
+};
+
+pub const CLIArgs = struct {
+    language: ?Language = null,
+    validate_release: bool = false,
+};
+
+pub fn main(shell: *Shell, gpa: std.mem.Allocator, cli_args: CLIArgs) !void {
+    if (cli_args.validate_release) {
+        try validate_release(shell, gpa, cli_args.language);
+    } else {
+        try generate_readmes(shell, gpa, cli_args.language);
+        try run_tests(shell, gpa, cli_args.language);
+    }
+}
+
+fn generate_readmes(shell: *Shell, gpa: std.mem.Allocator, language_requested: ?Language) !void {
+    inline for (comptime std.enums.values(Language)) |language| {
+        if (language_requested == language or language_requested == null) {
+            try shell.pushd("./src/clients/" ++ @tagName(language));
+            defer shell.popd();
+
+            try client_readmes.test_freshness(shell, gpa, language);
+        }
+    }
+}
+
+fn run_tests(shell: *Shell, gpa: std.mem.Allocator, language_requested: ?Language) !void {
+    inline for (comptime std.enums.values(Language)) |language| {
+        if (language_requested == language or language_requested == null) {
+            {
+                const ci = @field(LanguageCI, @tagName(language));
+                var section = try shell.open_section(@tagName(language) ++ " ci");
+                defer section.close();
+
+                {
+                    try shell.pushd("./src/clients/" ++ @tagName(language));
+                    defer shell.popd();
+
+                    try ci.tests(shell, gpa);
+                }
+            }
+
+            // Test the vortex drivers.
+            // These may expect the above driver tests to have run,
+            // in order to build the driver.
+            // They expect the vortex and tigerbeetle drivers to be built.
+            if (@hasField(@TypeOf(LanguageCIVortex), @tagName(language))) {
+                const ci = @field(LanguageCIVortex, @tagName(language));
+                var section = try shell.open_section(@tagName(language) ++ " vortex ci");
+                defer section.close();
+
+                {
+                    try shell.pushd("./src/testing/vortex/" ++ @tagName(language) ++ "_driver");
+                    defer shell.popd();
+
+                    try ci.tests(shell, gpa);
+                }
+            }
+        }
+    }
+}
+
+fn validate_release(shell: *Shell, gpa: std.mem.Allocator, language_requested: ?Language) !void {
+    var tmp_dir = std.testing.tmpDir(.{});
+    defer tmp_dir.cleanup();
+
+    try shell.pushd_dir(tmp_dir.dir);
+    defer shell.popd();
+
+    const release_info = try shell.exec_stdout(
+        "gh release --repo tigerbeetle/tigerbeetle list --limit 1",
+        .{},
+    );
+    const tag, _ = stdx.cut(release_info, "\t").?;
+    log.info("validating release {s}", .{tag});
+
+    try shell.exec(
+        "gh release --repo tigerbeetle/tigerbeetle download {tag}",
+        .{ .tag = tag },
+    );
+
+    if (builtin.os.tag != .linux) {
+        log.warn("skip release verification for platforms other than Linux", .{});
+    }
+
+    // Note: when updating the list of artifacts, don't forget to check for any external links.
+    //
+    // At minimum, `installation.md` requires an update.
+    const artifacts = [_][]const u8{
+        "tigerbeetle-aarch64-linux-debug.zip",
+        "tigerbeetle-aarch64-linux.zip",
+        "tigerbeetle-universal-macos-debug.zip",
+        "tigerbeetle-universal-macos.zip",
+        "tigerbeetle-x86_64-linux-debug.zip",
+        "tigerbeetle-x86_64-linux.zip",
+        "tigerbeetle-x86_64-windows-debug.zip",
+        "tigerbeetle-x86_64-windows.zip",
+    };
+    for (artifacts) |artifact| {
+        assert(shell.file_exists(artifact));
+    }
+
+    const git_sha = try shell.exec_stdout("git rev-parse HEAD", .{});
+    try shell.exec_zig_options(.{ .timeout = .minutes(20) }, "build scripts -- release --build " ++
+        "--sha={git_sha} --language=zig", .{
+        .git_sha = git_sha,
+    });
+    for (artifacts) |artifact| {
+        // Zig only guarantees release builds to be deterministic.
+        if (std.mem.indexOf(u8, artifact, "-debug.zip") != null) continue;
+
+        const checksum_downloaded = try shell.sha256sum(artifact);
+
+        shell.popd();
+        const checksum_built = try shell.sha256sum(try shell.fmt(
+            "zig-out/dist/tigerbeetle/{s}",
+            .{
+                artifact,
+            },
+        ));
+        try shell.pushd_dir(tmp_dir.dir);
+
+        if (checksum_downloaded != checksum_built) {
+            std.debug.panic("checksum mismatch - {s}: downloaded {x}, built {x}", .{
+                artifact,
+                checksum_downloaded,
+                checksum_built,
+            });
+        }
+    }
+
+    try shell.unzip_executable("tigerbeetle-x86_64-linux.zip", "tigerbeetle");
+
+    const version = try shell.exec_stdout("./tigerbeetle version --verbose", .{});
+    assert(std.mem.indexOf(u8, version, tag) != null);
+    assert(std.mem.indexOf(u8, version, "ReleaseSafe") != null);
+
+    const tigerbeetle_absolute_path = try shell.cwd.realpathAlloc(gpa, "tigerbeetle");
+    defer gpa.free(tigerbeetle_absolute_path);
+
+    inline for (comptime std.enums.values(Language)) |language| {
+        if (language_requested == language or language_requested == null) {
+            const ci = @field(LanguageCI, @tagName(language));
+            try ci.validate_release(shell, gpa, .{
+                .tigerbeetle = tigerbeetle_absolute_path,
+                .version = tag,
+            });
+        }
+    }
+
+    // Check all the client releases to ensure the latest published release is what it should be.
+    inline for (comptime std.enums.values(Language)) |language| {
+        if ((language == language_requested or language_requested == null) and
+            language != .rust) // Rust isn't published yet.
+        {
+            const ci = @field(LanguageCI, @tagName(language));
+            const release_published_latest = try ci.release_published_latest(shell);
+
+            if (!std.mem.eql(u8, release_published_latest, tag)) {
+                std.debug.panic("version mismatch - {s}: latest published {s}, expected {s}", .{
+                    @tagName(language),
+                    release_published_latest,
+                    tag,
+                });
+            }
+        }
+    }
+
+    // Check that the docker tag for latest is the same as the docker tag for the release. Docker's
+    // APIs make it much harder to do a release_published_latest() style check as above.
+    const docker_digest_latest = try docker_digest(shell, "latest");
+    const docker_digest_tagged = try docker_digest(shell, tag);
+
+    if (!std.mem.eql(u8, docker_digest_latest, docker_digest_tagged)) {
+        std.debug.panic("version mismatch - docker: latest published {s}, expected {s}", .{
+            docker_digest_latest,
+            docker_digest_tagged,
+        });
+    }
+
+    const docker_version = try shell.exec_stdout(
+        \\docker run ghcr.io/tigerbeetle/tigerbeetle:{version} version --verbose
+    , .{ .version = tag });
+    assert(std.mem.indexOf(u8, docker_version, tag) != null);
+    assert(std.mem.indexOf(u8, docker_version, "ReleaseSafe") != null);
+}
+
+fn docker_digest(shell: *Shell, version: []const u8) ![]const u8 {
+    try shell.exec(
+        \\docker pull ghcr.io/tigerbeetle/tigerbeetle:{version}
+    , .{ .version = version });
+
+    return try shell.exec_stdout("docker inspect --format={format} " ++
+        "ghcr.io/tigerbeetle/tigerbeetle:{version}", .{
+        .format = "{{index .RepoDigests 0}}",
+        .version = version,
+    });
+}