tigerbeetle 0.0.36 → 0.0.38

data/ext/tb_client/tigerbeetle/src/docs_website/src/file_checker.zig

@@ -0,0 +1,313 @@
+//! Sanity checks that all the generated files look reasonable.
+
+const std = @import("std");
+const log = std.log.scoped(.validate);
+const assert = std.debug.assert;
+
+const file_size_max = 166 * 1024;
+const search_index_size_max = 2000 * 1024;
+const single_page_size_max = 2000 * 1024;
+
+// If this is set to true, we check if we get a 200 response for any external links.
+const check_links_external: bool = false;
+
+var file_cache: std.StringHashMap([]const u8) = undefined;
+
+pub fn main() !void {
+    var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
+    const allocator = arena.allocator();
+    file_cache = std.StringHashMap([]const u8).init(allocator);
+    var args = std.process.args();
+    _ = args.skip();
+    const path = args.next().?;
+    assert(args.next() == null);
+    try validate_dir(allocator, path);
+}
+
+fn classify_file(path: []const u8) enum { text, binary, exception, unexpected } {
+    const text: []const []const u8 =
+        &.{ ".css", ".html", ".js", ".json", ".svg", ".xml" };
+    const binary: []const []const u8 =
+        &.{ ".avif", ".gif", ".jpg", ".png", ".ttf", ".webp", ".woff2" };
+    const exceptions: []const []const u8 =
+        &.{ "CNAME", ".nojekyll" };
+
+    const extension = std.fs.path.extension(path);
+    for (text) |text_extension| {
+        if (std.mem.eql(u8, extension, text_extension)) return .text;
+    }
+
+    for (binary) |binary_extension| {
+        if (std.mem.eql(u8, extension, binary_extension)) return .binary;
+    }
+
+    for (exceptions) |exception| {
+        if (std.mem.eql(u8, exception, path)) return .exception;
+    }
+
+    return .unexpected;
+}
+
+fn validate_dir(arena: std.mem.Allocator, path: []const u8) !void {
+    var dir = try std.fs.cwd().openDir(path, .{ .iterate = true });
+    defer dir.close();
+
+    var walker = try dir.walk(arena);
+    defer walker.deinit();
+
+    while (try walker.next()) |entry| switch (entry.kind) {
+        .file => try validate_file(.{ .arena = arena, .dir = dir, .path = entry.path }),
+        .directory => {},
+        else => {
+            log.err("unexpected file type: '{s}'", .{
+                try dir.realpathAlloc(arena, entry.path),
+            });
+            return error.UnsupportedFileType;
+        },
+    };
+}
+
+const FileValidationContext = struct {
+    arena: std.mem.Allocator,
+    dir: std.fs.Dir,
+    path: []const u8,
+};
+
+fn validate_file(context: FileValidationContext) !void {
+    const stat = context.dir.statFile(context.path) catch |err| {
+        log.err("unable to stat file '{s}': {s}", .{
+            try context.dir.realpathAlloc(context.arena, context.path),
+            @errorName(err),
+        });
+        return err;
+    };
+    const size_max: u64 = if (std.mem.eql(u8, context.path, "search-index.json"))
+        search_index_size_max
+    else if (std.mem.eql(u8, context.path, "single-page/index.html"))
+        single_page_size_max
+    else
+        file_size_max;
+    if (stat.size > size_max) {
+        log.err("file '{s}' with size {:.2} exceeds max file size of {:.2}", .{
+            try context.dir.realpathAlloc(context.arena, context.path),
+            std.fmt.fmtIntSizeBin(stat.size),
+            std.fmt.fmtIntSizeBin(size_max),
+        });
+        return error.FileSizeExceeded;
+    }
+
+    switch (classify_file(context.path)) {
+        .text => try validate_text_file(context),
+        .binary => {}, // Nothing to validate.
+        .exception => {}, // Nothing to validate.
+        .unexpected => {
+            log.err("file '{s}' has unsupported type '{s}'", .{
+                try context.dir.realpathAlloc(context.arena, context.path),
+                std.fs.path.extension(context.path),
+            });
+            return error.UnsupportedFileType;
+        },
+    }
+}
+
+fn validate_text_file(context: FileValidationContext) !void {
+    assert(classify_file(context.path) == .text);
+
+    const file = try context.dir.openFile(context.path, .{});
+    defer file.close();
+
+    try file.seekFromEnd(-1);
+    const last_byte = try file.reader().readByte();
+    if (last_byte != '\n') {
+        log.err("file '{s}' doesn't end with a newline", .{
+            try context.dir.realpathAlloc(context.arena, context.path),
+        });
+        return error.MissingNewline;
+    }
+
+    if (std.mem.endsWith(u8, context.path, ".html")) {
+        try check_links(context);
+    }
+}
+
+fn read_file_cached(arena: std.mem.Allocator, dir: std.fs.Dir, path: []const u8) ![]const u8 {
+    if (file_cache.get(path)) |content| return content;
+
+    const content = try dir.readFileAlloc(arena, path, 2 * 1024 * 1024);
+    try file_cache.put(try arena.dupe(u8, path), content);
+
+    return content;
+}
+
+// These links don't work with https.
+const http_exceptions = std.StaticStringMap(void).initComptime(.{
+    .{"http://www.bailis.org/blog/linearizability-versus-serializability/"},
+    .{"http://pmg.csail.mit.edu/papers/vr-revisited.pdf"},
+});
+
+// These links cause TLS errors with std.http.Client.
+const https_exceptions = std.StaticStringMap(void).initComptime(.{
+    .{"https://www.eecg.utoronto.ca/~yuan/papers/failure_analysis_osdi14.pdf"},
+    .{"https://pmg.csail.mit.edu/papers/vr.pdf"},
+    .{"https://www.infoq.com/presentations/LMAX/"},
+    .{"https://kernel.dk/io_uring.pdf"},
+    .{"https://research.cs.wisc.edu/wind/Publications/latent-sigmetrics07.pdf"},
+    .{"https://security.googleblog.com/2023/06/learnings-from-kctf-vrps-42-linux.html"},
+});
+
+fn check_links(context: FileValidationContext) !void {
+    const html = try read_file_cached(context.arena, context.dir, context.path);
+
+    var link_iterator = LinkIterator.init(html);
+    errdefer log.err("[link checker] error in {s}:{}", .{
+        context.dir.realpathAlloc(context.arena, context.path) catch unreachable,
+        link_iterator.line_number,
+    });
+
+    while (link_iterator.next()) |link| {
+        try check_link(context, link);
+    }
+}
+
+fn check_link(context: FileValidationContext, link: Link) !void {
+    // Check schema.
+    {
+        if (std.mem.startsWith(u8, link.base, "mailto:")) {
+            return; // Ignore.
+        }
+
+        if (std.mem.startsWith(u8, link.base, "https://")) {
+            return check_link_external(context.arena, link);
+        }
+
+        if (std.mem.startsWith(u8, link.base, "http://")) {
+            if (http_exceptions.has(link.base)) {
+                return check_link_external(context.arena, link);
+            }
+
+            log.err("found insecure link: '{s}'", .{link.base});
+            return error.InsecureLink;
+        }
+    }
+
+    if (std.mem.indexOf(u8, link.base, "//") != null or
+        std.mem.indexOf(u8, link.base, "/./") != null)
+    {
+        log.err("redundant slash: '{s}'", .{link.base});
+        return error.RedundantSlash;
+    }
+
+    // Locate local link target.
+    var target = link.base;
+    const is_absolute = target.len > 0 and target[0] == '/';
+    if (is_absolute) {
+        target = target[1..];
+    } else if (std.fs.path.dirname(context.path)) |dirname| {
+        target = try std.fs.path.join(context.arena, &.{ dirname, target });
+    }
+
+    const is_directory = std.fs.path.extension(target).len == 0;
+    if (is_directory) {
+        target = try std.fs.path.join(context.arena, &.{ target, "index.html" });
+    }
+
+    if (!try path_exists(context.dir, target)) {
+        log.err("link target not found: '{s}'", .{target});
+        return error.TargetNotFound;
+    }
+
+    if (link.fragment) |fragment| {
+        try check_link_fragment(context, target, fragment);
+    }
+}
+
+fn check_link_external(arena: std.mem.Allocator, link: Link) !void {
+    if (!check_links_external) return;
+    if (https_exceptions.has(link.base)) return;
+
+    errdefer |err| log.err("got {} while checking external link '{s}'", .{ err, link.base });
+
+    log.info("checking external link '{s}'", .{link.base});
+
+    var client = std.http.Client{ .allocator = arena };
+    defer client.deinit();
+
+    const uri = try std.Uri.parse(link.base);
+    var header_buffer: [512 * 1024]u8 = undefined;
+    var request = try client.open(.GET, uri, .{ .server_header_buffer = &header_buffer });
+    defer request.deinit();
+
+    try request.send();
+    try request.finish();
+    try request.wait();
+
+    if (request.response.status != std.http.Status.ok) {
+        return error.WrongStatusResponse;
+    }
+}
+
+fn check_link_fragment(
+    context: FileValidationContext,
+    target_path: []const u8,
+    fragment: []const u8,
+) !void {
+    assert(std.mem.endsWith(u8, target_path, ".html"));
+
+    const html = try read_file_cached(context.arena, context.dir, target_path);
+    const needle = try std.mem.concat(context.arena, u8, &.{ "id=\"", fragment, "\"" });
+    if (std.mem.indexOf(u8, html, needle) == null) {
+        log.err("link target '{s}' does not contain anchor: '{s}'", .{ target_path, fragment });
+        return error.AnchorNotFound;
+    }
+}
+
+const Link = struct {
+    base: []const u8,
+    fragment: ?[]const u8 = null,
+
+    fn parse(text: []const u8) Link {
+        if (std.mem.lastIndexOfScalar(u8, text, '#')) |index| {
+            return .{
+                .base = text[0..index],
+                .fragment = text[index + 1 ..],
+            };
+        }
+        return .{ .base = text };
+    }
+};
+
+const LinkIterator = struct {
+    line_number: u32 = 1,
+    remaining: []const u8,
+
+    const href_prefix = "href=\"";
+
+    fn init(html: []const u8) LinkIterator {
+        return .{ .remaining = html };
+    }
+
+    fn next(self: *LinkIterator) ?Link {
+        const index = std.mem.indexOf(u8, self.remaining, href_prefix) orelse
+            return null;
+        const uri_start = index + href_prefix.len;
+        const uri_len = std.mem.indexOfScalar(u8, self.remaining[uri_start..], '"') orelse
+            return null;
+        const uri_end = uri_start + uri_len;
+        const uri_text = self.remaining[uri_start..][0..uri_len];
+
+        for (self.remaining[0..uri_start]) |c| {
+            if (c == '\n') self.line_number += 1;
+        }
+        self.remaining = self.remaining[uri_end..];
+
+        return Link.parse(uri_text);
+    }
+};
+
+fn path_exists(dir: std.fs.Dir, path: []const u8) !bool {
+    dir.access(path, .{}) catch |err| switch (err) {
+        error.FileNotFound => return false,
+        else => return err,
+    };
+    return true;
+}

data/ext/tb_client/tigerbeetle/src/docs_website/src/html.zig

@@ -0,0 +1,87 @@
+const std = @import("std");
+const log = std.log.scoped(.template);
+
+pub const Html = @This();
+
+arena: std.mem.Allocator,
+buffer: std.ArrayList(u8),
+writer: std.ArrayList(u8).Writer,
+
+pub fn create(arena: std.mem.Allocator) !*Html {
+    var html = try arena.create(Html);
+    html.* = .{
+        .arena = arena,
+        .buffer = std.ArrayList(u8).init(arena),
+        .writer = undefined,
+    };
+    html.writer = html.buffer.writer();
+    return html;
+}
+
+/// Replaces the variables in `$snake_case` matching the names of the fields in the
+/// `replacement` struct.
+///
+/// We picked the $dollar_name format over a {curly_braces} format to avoid potential ambiguities
+/// with JavaScript function syntax in the template.
+pub fn write(html: *Html, template: []const u8, replacements: anytype) !void {
+    const ReplacementsType = @TypeOf(replacements);
+    const replacements_type_info = @typeInfo(ReplacementsType);
+    if (replacements_type_info != .@"struct") @compileError("expected struct");
+
+    var unused = std.StringHashMap(void).init(html.arena);
+    inline for (replacements_type_info.@"struct".fields) |field| {
+        try unused.put(field.name, {});
+    }
+
+    var it = std.mem.tokenizeScalar(u8, template, '$');
+    if (template[0] != '$') if (it.next()) |prefix| try html.writer.writeAll(prefix);
+    while (it.next()) |chunk| {
+        const identifier_len = for (chunk, 0..) |c, index| {
+            switch (c) {
+                'a'...'z', '_' => {},
+                else => break index,
+            }
+        } else chunk.len;
+
+        const identifier = chunk[0..identifier_len];
+        const found = inline for (replacements_type_info.@"struct".fields) |field| {
+            if (std.mem.eql(u8, field.name, identifier)) {
+                try html.writer.writeAll(switch (field.type) {
+                    *Html => @field(replacements, field.name).string(),
+                    else => @field(replacements, field.name),
+                });
+                _ = unused.remove(field.name);
+                break true;
+            }
+        } else false;
+        if (!found) {
+            log.err("Html.write: identifier '{s}' not found in replacements", .{identifier});
+            return error.IdentifierNotFound;
+        }
+
+        try html.writer.writeAll(chunk[identifier_len..]);
+    }
+
+    if (unused.count() > 0) {
+        var unused_it = unused.keyIterator();
+        while (unused_it.next()) |unused_identifier| {
+            log.err("Html.write: identifier '{s}' not found in template", .{unused_identifier.*});
+        }
+        return error.UnusedIdentifiers;
+    }
+}
+
+pub fn child(self: Html) !*Html {
+    return try Html.create(self.arena);
+}
+
+pub fn string(self: Html) []const u8 {
+    return self.buffer.items;
+}
+
+pub fn redirect(arena: std.mem.Allocator, url: []const u8) ![]const u8 {
+    const template = @embedFile("html/redirect.html");
+    var html = try Html.create(arena);
+    try html.write(template, .{ .url = url });
+    return html.string();
+}

data/ext/tb_client/tigerbeetle/src/docs_website/src/page_writer.zig

@@ -0,0 +1,63 @@
+const std = @import("std");
+const assert = std.debug.assert;
+const Website = @import("website.zig").Website;
+const Html = @import("html.zig").Html;
+
+const page_template = @embedFile("html/page.html");
+const search_box_template = @embedFile("html/search-box.html");
+const search_results_template = @embedFile("html/search-results.html");
+const search_script_template = "<script src=\"$url_prefix/js/search.js\"></script>";
+const page_script = @embedFile("js/page-script.js");
+
+pub fn main() !void {
+    var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
+    const allocator = arena.allocator();
+    const args = try std.process.argsAlloc(allocator);
+    assert(args.len == 9);
+    const title = args[1];
+    const author = args[2];
+    const url_prefix = args[3];
+    const page_path = args[4];
+    const include_search = std.mem.eql(u8, args[5], "true");
+    const nav = args[6];
+    const source_file_path = args[7];
+    const target_file_path = args[8];
+
+    var script = try Html.create(allocator);
+    try script.write(page_script, .{ .url_prefix = url_prefix });
+
+    var script_hash: [32]u8 = undefined;
+    std.crypto.hash.sha2.Sha256.hash(script.string(), &script_hash, .{});
+    const b64_encoder = std.base64.standard.Encoder;
+    var script_hash_b64_buf: [b64_encoder.calcSize(script_hash.len)]u8 = undefined;
+    const script_hash_b64 = b64_encoder.encode(&script_hash_b64_buf, &script_hash);
+
+    const content = try std.fs.cwd().readFileAlloc(
+        allocator,
+        source_file_path,
+        Website.file_size_max,
+    );
+    var html = try Html.create(allocator);
+    var search_box = try html.child();
+    var search_results = try html.child();
+    var search_script = try html.child();
+    if (include_search) {
+        try search_box.write(search_box_template, .{});
+        try search_results.write(search_results_template, .{ .url_prefix = url_prefix });
+        try search_script.write(search_script_template, .{ .url_prefix = url_prefix });
+    }
+    try html.write(page_template, .{
+        .page_script_hash = script_hash_b64,
+        .title = title,
+        .author = author,
+        .url_prefix = url_prefix,
+        .page_path = page_path,
+        .nav = nav,
+        .search_box = search_box,
+        .search_results = search_results,
+        .content = content,
+        .page_script = script,
+        .search_script = search_script,
+    });
+    try std.fs.cwd().writeFile(.{ .sub_path = target_file_path, .data = html.string() });
+}

data/ext/tb_client/tigerbeetle/src/docs_website/src/redirects.zig

@@ -0,0 +1,47 @@
+const std = @import("std");
+
+const Website = @import("website.zig").Website;
+
+const Redirect = struct {
+    old: []const u8,
+    new: []const u8,
+
+    const all: []const Redirect = &.{
+        .{ .old = "quick-start/", .new = "start/" },
+        .{ .old = "about/", .new = "concepts/" },
+        .{ .old = "about/vopr/", .new = "concepts/safety/" },
+        .{ .old = "about/oltp/", .new = "concepts/oltp/" },
+    };
+};
+
+pub fn build(b: *std.Build, content: *std.Build.Step.WriteFile, website: Website) !void {
+    for (Redirect.all) |redirect| {
+        try build_redirect(b, content, website, redirect);
+    }
+}
+
+fn build_redirect(
+    b: *std.Build,
+    content: *std.Build.Step.WriteFile,
+    website: Website,
+    redirect: Redirect,
+) !void {
+    const path = b.pathJoin(&.{ redirect.old, "index.html" });
+    const url = b.fmt("{s}/{s}", .{ website.url_prefix, redirect.new });
+    const html_redirect = b.fmt(
+        \\<!DOCTYPE html>
+        \\<html lang="en">
+        \\  <meta charset="utf-8">
+        \\  <title>Redirecting&hellip;</title>
+        \\  <link rel="canonical" href="{[url]s}">
+        \\  <script>location="{[url]s}"</script>
+        \\  <meta http-equiv="refresh" content="0; url={[url]s}">
+        \\  <meta name="robots" content="noindex">
+        \\  <h1>Redirecting&hellip;</h1>
+        \\  <a href="{[url]s}">Click here if you are not redirected.</a>
+        \\</html>
+        \\
+    , .{ .url = url });
+
+    _ = content.add(path, html_redirect);
+}

data/ext/tb_client/tigerbeetle/src/docs_website/src/search_index_writer.zig

@@ -0,0 +1,28 @@
+const std = @import("std");
+const Website = @import("website.zig").Website;
+
+const Entry = struct {
+    path: []const u8,
+    html: []const u8,
+};
+
+pub fn main() !void {
+    var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
+    const allocator = arena.allocator();
+
+    var args = std.process.args();
+    _ = args.skip();
+
+    var entries = std.ArrayList(Entry).init(allocator);
+    while (args.next()) |path| {
+        const html = args.next().?;
+        const entry = Entry{
+            .path = path,
+            .html = try std.fs.cwd().readFileAlloc(allocator, html, Website.file_size_max),
+        };
+        try entries.append(entry);
+    }
+
+    const json_string = try std.json.stringifyAlloc(allocator, entries.items, .{});
+    try std.io.getStdOut().writer().print("{s}\n", .{json_string});
+}

data/ext/tb_client/tigerbeetle/src/docs_website/src/service_worker_writer.zig

@@ -0,0 +1,61 @@
+const std = @import("std");
+const Html = @import("html.zig").Html;
+
+pub fn main() !void {
+    var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
+    const allocator = arena.allocator();
+    var args = std.process.args();
+    _ = args.skip();
+    const url_prefix = args.next().?;
+    const cache_name = args.next().?;
+    const search_path = args.next().?;
+
+    const file_paths = try collect_files(allocator, url_prefix, search_path);
+    try write_service_worker(allocator, cache_name, file_paths);
+}
+
+fn collect_files(
+    arena: std.mem.Allocator,
+    url_prefix: []const u8,
+    search_path: []const u8,
+) ![]const []const u8 {
+    var file_paths = std.ArrayList([]const u8).init(arena);
+
+    var dir = try std.fs.cwd().openDir(search_path, .{ .iterate = true });
+    defer dir.close();
+
+    var walker = try dir.walk(arena);
+    defer walker.deinit();
+
+    while (try walker.next()) |entry| {
+        if (entry.kind == .file) {
+            // Normalize requests by using directory with trailing slash instead of index.html.
+            if (std.mem.endsWith(u8, entry.path, "index.html")) {
+                const stripped = entry.path[0 .. entry.path.len - "index.html".len];
+                try file_paths.append(try std.mem.join(arena, "/", &.{ url_prefix, stripped }));
+            } else {
+                try file_paths.append(try std.mem.join(arena, "/", &.{ url_prefix, entry.path }));
+            }
+        }
+    }
+
+    return file_paths.toOwnedSlice();
+}
+
+fn write_service_worker(
+    arena: std.mem.Allocator,
+    cache_name: []const u8,
+    file_paths: []const []const u8,
+) !void {
+    const template = @embedFile("js/service-worker.js");
+
+    const file_paths_json = try std.json.stringifyAlloc(arena, file_paths, .{});
+
+    var html = try Html.create(arena);
+    try html.write(template, .{
+        .cache_name = cache_name,
+        .files_to_cache = file_paths_json,
+    });
+
+    try std.io.getStdOut().writer().print("{s}", .{html.string()});
+}