tigerbeetle 0.0.34 → 0.0.37

data/ext/tb_client/tigerbeetle/src/docs_website/src/docs.zig

@@ -0,0 +1,252 @@
+const std = @import("std");
+const Allocator = std.mem.Allocator;
+const LazyPath = std.Build.LazyPath;
+const Website = @import("website.zig").Website;
+const Html = @import("html.zig").Html;
+const content = @import("./content.zig");
+
+const base_path = "../../docs";
+
+const SearchIndexEntry = struct {
+    page_path: []const u8,
+    html_path: LazyPath,
+};
+const SearchIndex = std.ArrayList(SearchIndexEntry);
+
+pub fn build(
+    b: *std.Build,
+    output: *std.Build.Step.WriteFile,
+    website: Website,
+) !void {
+    const arena = b.allocator;
+
+    var search_index = SearchIndex.init(arena);
+
+    var page_buffer: [1 << 16]u8 = undefined;
+    var base = try b.build_root.handle.openDir(base_path, .{});
+    defer base.close();
+
+    const root_page = try content.load(arena, base, &page_buffer);
+
+    try tree_install(b, website, output, &search_index, root_page, root_page);
+
+    const run_search_index_writer = b.addRunArtifact(b.addExecutable(.{
+        .name = "search_index_writer",
+        .root_source_file = b.path("src/search_index_writer.zig"),
+        .target = b.graph.host,
+    }));
+    for (search_index.items) |entry| {
+        run_search_index_writer.addArg(entry.page_path);
+        run_search_index_writer.addFileArg(entry.html_path);
+    }
+    _ = output.addCopyFile(
+        run_search_index_writer.captureStdOut(),
+        "search-index.json",
+    );
+
+    try write_single_page(b, website, &search_index, root_page, output);
+
+    try write_404_page(b, website, output);
+}
+
+fn tree_install(
+    b: *std.Build,
+    website: Website,
+    output: *std.Build.Step.WriteFile,
+    search_index: *SearchIndex,
+    root: content.Page,
+    page: content.Page,
+) !void {
+    try page_install(b, website, output, search_index, root, page);
+    for (page.children) |child| try tree_install(b, website, output, search_index, root, child);
+}
+
+fn page_install(
+    b: *std.Build,
+    website: Website,
+    output: *std.Build.Step.WriteFile,
+    search_index: *SearchIndex,
+    root: content.Page,
+    page: content.Page,
+) !void {
+    const page_html = run_pandoc(b, website.pandoc_bin, page.path);
+
+    const page_path = page_url(b.allocator, page);
+
+    try search_index.append(.{
+        .page_path = page_path,
+        .html_path = page_html,
+    });
+
+    const nav_html = try Html.create(b.allocator);
+    try nav_fill(website, nav_html, root, .{ .target = page });
+
+    try nav_html.write(
+        @embedFile("html/single-page-link.html"),
+        .{ .url_prefix = website.url_prefix },
+    );
+
+    // Add trailing slash.
+    const page_path_canonical = if (page_path.len == 0) "" else b.fmt("{s}/", .{page_path});
+
+    const page_out = website.write_page(.{
+        .title = page.content.title,
+        .page_path = page_path_canonical,
+        .nav = nav_html.string(),
+        .content = page_html,
+    });
+    _ = output.addCopyFile(page_out, b.pathJoin(&.{ page_path, "index.html" }));
+}
+
+fn page_url(arena: Allocator, page: content.Page) []const u8 {
+    const url = cut_suffix(page.path, "/README.md") orelse cut_suffix(page.path, ".md").?;
+    if (cut_prefix(url, "../src/clients/")) |client| {
+        // Special case: docs for clients are in `/src/clients/$lang`, not under `/docs`.
+        return std.mem.concat(arena, u8, &.{ "coding/clients/", client }) catch @panic("OOM");
+    }
+
+    if (url.len == 1 and url[0] == '.') return "";
+    return cut_prefix(url, "./").?;
+}
+
+fn url2slug(arena: Allocator, url: []const u8) []const u8 {
+    const slug = arena.dupe(u8, url) catch @panic("OOM");
+    std.mem.replaceScalar(u8, slug, '/', '-');
+    return std.mem.concat(arena, u8, &.{ "#", slug }) catch @panic("OOM");
+}
+
+fn nav_fill(website: Website, html: *Html, node: content.Page, options: struct {
+    target: content.Page,
+    single_page: bool = false,
+}) !void {
+    try html.write("<ol>\n", .{});
+    for (node.children, node.content.children) |node_child, content_child| {
+        var url = page_url(html.arena, node_child);
+        if (options.single_page) {
+            url = url2slug(html.arena, url);
+        } else {
+            url = try std.fmt.allocPrint(html.arena, "{s}/{s}/", .{ website.url_prefix, url });
+        }
+
+        if (node_child.children.len > 0) {
+            try html.write("<li>\n<details", .{});
+            if (nav_contains(node_child, options.target)) try html.write(" open", .{});
+            try html.write("><summary class=\"item\">", .{});
+            try html.write(
+                \\<a href="$url">$title</a>
+            , .{
+                .url = url,
+                // Fabio: index page titles are too long
+                .title = content_child.title,
+            });
+            try html.write("</summary>\n", .{});
+            try nav_fill(website, html, node_child, options);
+            try html.write("</details></li>\n", .{});
+        } else {
+            try html.write(
+                \\<li class="item"><a href="$url"$class>$title</a></li>
+                \\
+            , .{
+                .url = url,
+                .class = if (nav_same_page(node_child, options.target)) " class=\"target\"" else "",
+                .title = content_child.title,
+            });
+        }
+    }
+    try html.write("</ol>\n", .{});
+}
+
+fn nav_contains(node: content.Page, target: content.Page) bool {
+    if (nav_same_page(node, target)) return true;
+    for (node.children) |child| {
+        if (nav_contains(child, target)) return true;
+    }
+    return false;
+}
+
+fn nav_same_page(a: content.Page, b: content.Page) bool {
+    return std.mem.eql(u8, a.path, b.path);
+}
+
+fn run_pandoc(
+    b: *std.Build,
+    pandoc_bin: std.Build.LazyPath,
+    source: []const u8,
+) std.Build.LazyPath {
+    const pandoc_step = std.Build.Step.Run.create(b, "run pandoc");
+    pandoc_step.addFileArg(pandoc_bin);
+    pandoc_step.addArgs(&.{ "--from", "gfm+smart-tex_math_dollars", "--to", "html5" });
+    pandoc_step.addPrefixedFileArg("--lua-filter=", b.path("pandoc/markdown-links.lua"));
+    pandoc_step.addPrefixedFileArg("--lua-filter=", b.path("pandoc/anchor-links.lua"));
+    pandoc_step.addPrefixedFileArg("--lua-filter=", b.path("pandoc/table-wrapper.lua"));
+    pandoc_step.addPrefixedFileArg("--lua-filter=", b.path("pandoc/code-block-buttons.lua"));
+    pandoc_step.addPrefixedFileArg("--lua-filter=", b.path("pandoc/edit-link-footer.lua"));
+    pandoc_step.addArg("--reference-location=section");
+    const result = pandoc_step.addPrefixedOutputFileArg("--output=", "pandoc-out.html");
+    pandoc_step.addFileArg(b.path(base_path).path(b, source));
+    return result;
+}
+
+fn write_single_page(
+    b: *std.Build,
+    website: Website,
+    search_index: *const SearchIndex,
+    root: content.Page,
+    docs: *std.Build.Step.WriteFile,
+) !void {
+    const run_single_page_writer = b.addRunArtifact(b.addExecutable(.{
+        .name = "single_page_writer",
+        .root_source_file = b.path("src/single_page_writer.zig"),
+        .target = b.graph.host,
+    }));
+    for (search_index.items) |entry| {
+        run_single_page_writer.addArg(entry.page_path);
+        run_single_page_writer.addFileArg(entry.html_path);
+    }
+    const nav_html = try Html.create(b.allocator);
+    try nav_fill(website, nav_html, root, .{ .target = root, .single_page = true });
+
+    const single_page = website.write_page(.{
+        .page_path = "single-page/",
+        .include_search = false,
+        .nav = nav_html.string(),
+        .content = run_single_page_writer.captureStdOut(),
+    });
+
+    _ = docs.addCopyFile(single_page, "single-page/index.html");
+}
+
+fn write_404_page(
+    b: *std.Build,
+    website: Website,
+    docs: *std.Build.Step.WriteFile,
+) !void {
+    const template = @embedFile("html/404.html");
+    var html = try Html.create(b.allocator);
+    try html.write(template, .{
+        .url_prefix = website.url_prefix,
+        .title = "Page not found",
+        .author = "TigerBeetle Team",
+    });
+    _ = docs.add("404.html", html.string());
+}
+
+pub fn cut(haystack: []const u8, needle: []const u8) ?struct { []const u8, []const u8 } {
+    const index = std.mem.indexOf(u8, haystack, needle) orelse return null;
+
+    return .{ haystack[0..index], haystack[index + needle.len ..] };
+}
+
+pub fn cut_prefix(text: []const u8, comptime prefix: []const u8) ?[]const u8 {
+    return if (std.mem.startsWith(u8, text, prefix))
+        text[prefix.len..]
+    else
+        null;
+}
+
+pub fn cut_suffix(text: []const u8, comptime suffix: []const u8) ?[]const u8 {
+    return if (std.mem.endsWith(u8, text, suffix))
+        text[0 .. text.len - suffix.len]
+    else
+        null;
+}

data/ext/tb_client/tigerbeetle/src/docs_website/src/file_checker.zig

@@ -0,0 +1,313 @@
+//! Sanity checks that all the generated files look reasonable.
+
+const std = @import("std");
+const log = std.log.scoped(.validate);
+const assert = std.debug.assert;
+
+const file_size_max = 166 * 1024;
+const search_index_size_max = 2000 * 1024;
+const single_page_size_max = 2000 * 1024;
+
+// If this is set to true, we check if we get a 200 response for any external links.
+const check_links_external: bool = false;
+
+var file_cache: std.StringHashMap([]const u8) = undefined;
+
+pub fn main() !void {
+    var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
+    const allocator = arena.allocator();
+    file_cache = std.StringHashMap([]const u8).init(allocator);
+    var args = std.process.args();
+    _ = args.skip();
+    const path = args.next().?;
+    assert(args.next() == null);
+    try validate_dir(allocator, path);
+}
+
+fn classify_file(path: []const u8) enum { text, binary, exception, unexpected } {
+    const text: []const []const u8 =
+        &.{ ".css", ".html", ".js", ".json", ".svg", ".xml" };
+    const binary: []const []const u8 =
+        &.{ ".avif", ".gif", ".jpg", ".png", ".ttf", ".webp", ".woff2" };
+    const exceptions: []const []const u8 =
+        &.{ "CNAME", ".nojekyll" };
+
+    const extension = std.fs.path.extension(path);
+    for (text) |text_extension| {
+        if (std.mem.eql(u8, extension, text_extension)) return .text;
+    }
+
+    for (binary) |binary_extension| {
+        if (std.mem.eql(u8, extension, binary_extension)) return .binary;
+    }
+
+    for (exceptions) |exception| {
+        if (std.mem.eql(u8, exception, path)) return .exception;
+    }
+
+    return .unexpected;
+}
+
+fn validate_dir(arena: std.mem.Allocator, path: []const u8) !void {
+    var dir = try std.fs.cwd().openDir(path, .{ .iterate = true });
+    defer dir.close();
+
+    var walker = try dir.walk(arena);
+    defer walker.deinit();
+
+    while (try walker.next()) |entry| switch (entry.kind) {
+        .file => try validate_file(.{ .arena = arena, .dir = dir, .path = entry.path }),
+        .directory => {},
+        else => {
+            log.err("unexpected file type: '{s}'", .{
+                try dir.realpathAlloc(arena, entry.path),
+            });
+            return error.UnsupportedFileType;
+        },
+    };
+}
+
+const FileValidationContext = struct {
+    arena: std.mem.Allocator,
+    dir: std.fs.Dir,
+    path: []const u8,
+};
+
+fn validate_file(context: FileValidationContext) !void {
+    const stat = context.dir.statFile(context.path) catch |err| {
+        log.err("unable to stat file '{s}': {s}", .{
+            try context.dir.realpathAlloc(context.arena, context.path),
+            @errorName(err),
+        });
+        return err;
+    };
+    const size_max: u64 = if (std.mem.eql(u8, context.path, "search-index.json"))
+        search_index_size_max
+    else if (std.mem.eql(u8, context.path, "single-page/index.html"))
+        single_page_size_max
+    else
+        file_size_max;
+    if (stat.size > size_max) {
+        log.err("file '{s}' with size {:.2} exceeds max file size of {:.2}", .{
+            try context.dir.realpathAlloc(context.arena, context.path),
+            std.fmt.fmtIntSizeBin(stat.size),
+            std.fmt.fmtIntSizeBin(size_max),
+        });
+        return error.FileSizeExceeded;
+    }
+
+    switch (classify_file(context.path)) {
+        .text => try validate_text_file(context),
+        .binary => {}, // Nothing to validate.
+        .exception => {}, // Nothing to validate.
+        .unexpected => {
+            log.err("file '{s}' has unsupported type '{s}'", .{
+                try context.dir.realpathAlloc(context.arena, context.path),
+                std.fs.path.extension(context.path),
+            });
+            return error.UnsupportedFileType;
+        },
+    }
+}
+
+fn validate_text_file(context: FileValidationContext) !void {
+    assert(classify_file(context.path) == .text);
+
+    const file = try context.dir.openFile(context.path, .{});
+    defer file.close();
+
+    try file.seekFromEnd(-1);
+    const last_byte = try file.reader().readByte();
+    if (last_byte != '\n') {
+        log.err("file '{s}' doesn't end with a newline", .{
+            try context.dir.realpathAlloc(context.arena, context.path),
+        });
+        return error.MissingNewline;
+    }
+
+    if (std.mem.endsWith(u8, context.path, ".html")) {
+        try check_links(context);
+    }
+}
+
+fn read_file_cached(arena: std.mem.Allocator, dir: std.fs.Dir, path: []const u8) ![]const u8 {
+    if (file_cache.get(path)) |content| return content;
+
+    const content = try dir.readFileAlloc(arena, path, 2 * 1024 * 1024);
+    try file_cache.put(try arena.dupe(u8, path), content);
+
+    return content;
+}
+
+// These links don't work with https.
+const http_exceptions = std.StaticStringMap(void).initComptime(.{
+    .{"http://www.bailis.org/blog/linearizability-versus-serializability/"},
+    .{"http://pmg.csail.mit.edu/papers/vr-revisited.pdf"},
+});
+
+// These links cause TLS errors with std.http.Client.
+const https_exceptions = std.StaticStringMap(void).initComptime(.{
+    .{"https://www.eecg.utoronto.ca/~yuan/papers/failure_analysis_osdi14.pdf"},
+    .{"https://pmg.csail.mit.edu/papers/vr.pdf"},
+    .{"https://www.infoq.com/presentations/LMAX/"},
+    .{"https://kernel.dk/io_uring.pdf"},
+    .{"https://research.cs.wisc.edu/wind/Publications/latent-sigmetrics07.pdf"},
+    .{"https://security.googleblog.com/2023/06/learnings-from-kctf-vrps-42-linux.html"},
+});
+
+fn check_links(context: FileValidationContext) !void {
+    const html = try read_file_cached(context.arena, context.dir, context.path);
+
+    var link_iterator = LinkIterator.init(html);
+    errdefer log.err("[link checker] error in {s}:{}", .{
+        context.dir.realpathAlloc(context.arena, context.path) catch unreachable,
+        link_iterator.line_number,
+    });
+
+    while (link_iterator.next()) |link| {
+        try check_link(context, link);
+    }
+}
+
+fn check_link(context: FileValidationContext, link: Link) !void {
+    // Check schema.
+    {
+        if (std.mem.startsWith(u8, link.base, "mailto:")) {
+            return; // Ignore.
+        }
+
+        if (std.mem.startsWith(u8, link.base, "https://")) {
+            return check_link_external(context.arena, link);
+        }
+
+        if (std.mem.startsWith(u8, link.base, "http://")) {
+            if (http_exceptions.has(link.base)) {
+                return check_link_external(context.arena, link);
+            }
+
+            log.err("found insecure link: '{s}'", .{link.base});
+            return error.InsecureLink;
+        }
+    }
+
+    if (std.mem.indexOf(u8, link.base, "//") != null or
+        std.mem.indexOf(u8, link.base, "/./") != null)
+    {
+        log.err("redundant slash: '{s}'", .{link.base});
+        return error.RedundantSlash;
+    }
+
+    // Locate local link target.
+    var target = link.base;
+    const is_absolute = target.len > 0 and target[0] == '/';
+    if (is_absolute) {
+        target = target[1..];
+    } else if (std.fs.path.dirname(context.path)) |dirname| {
+        target = try std.fs.path.join(context.arena, &.{ dirname, target });
+    }
+
+    const is_directory = std.fs.path.extension(target).len == 0;
+    if (is_directory) {
+        target = try std.fs.path.join(context.arena, &.{ target, "index.html" });
+    }
+
+    if (!try path_exists(context.dir, target)) {
+        log.err("link target not found: '{s}'", .{target});
+        return error.TargetNotFound;
+    }
+
+    if (link.fragment) |fragment| {
+        try check_link_fragment(context, target, fragment);
+    }
+}
+
+fn check_link_external(arena: std.mem.Allocator, link: Link) !void {
+    if (!check_links_external) return;
+    if (https_exceptions.has(link.base)) return;
+
+    errdefer |err| log.err("got {} while checking external link '{s}'", .{ err, link.base });
+
+    log.info("checking external link '{s}'", .{link.base});
+
+    var client = std.http.Client{ .allocator = arena };
+    defer client.deinit();
+
+    const uri = try std.Uri.parse(link.base);
+    var header_buffer: [512 * 1024]u8 = undefined;
+    var request = try client.open(.GET, uri, .{ .server_header_buffer = &header_buffer });
+    defer request.deinit();
+
+    try request.send();
+    try request.finish();
+    try request.wait();
+
+    if (request.response.status != std.http.Status.ok) {
+        return error.WrongStatusResponse;
+    }
+}
+
+fn check_link_fragment(
+    context: FileValidationContext,
+    target_path: []const u8,
+    fragment: []const u8,
+) !void {
+    assert(std.mem.endsWith(u8, target_path, ".html"));
+
+    const html = try read_file_cached(context.arena, context.dir, target_path);
+    const needle = try std.mem.concat(context.arena, u8, &.{ "id=\"", fragment, "\"" });
+    if (std.mem.indexOf(u8, html, needle) == null) {
+        log.err("link target '{s}' does not contain anchor: '{s}'", .{ target_path, fragment });
+        return error.AnchorNotFound;
+    }
+}
+
+const Link = struct {
+    base: []const u8,
+    fragment: ?[]const u8 = null,
+
+    fn parse(text: []const u8) Link {
+        if (std.mem.lastIndexOfScalar(u8, text, '#')) |index| {
+            return .{
+                .base = text[0..index],
+                .fragment = text[index + 1 ..],
+            };
+        }
+        return .{ .base = text };
+    }
+};
+
+const LinkIterator = struct {
+    line_number: u32 = 1,
+    remaining: []const u8,
+
+    const href_prefix = "href=\"";
+
+    fn init(html: []const u8) LinkIterator {
+        return .{ .remaining = html };
+    }
+
+    fn next(self: *LinkIterator) ?Link {
+        const index = std.mem.indexOf(u8, self.remaining, href_prefix) orelse
+            return null;
+        const uri_start = index + href_prefix.len;
+        const uri_len = std.mem.indexOfScalar(u8, self.remaining[uri_start..], '"') orelse
+            return null;
+        const uri_end = uri_start + uri_len;
+        const uri_text = self.remaining[uri_start..][0..uri_len];
+
+        for (self.remaining[0..uri_start]) |c| {
+            if (c == '\n') self.line_number += 1;
+        }
+        self.remaining = self.remaining[uri_end..];
+
+        return Link.parse(uri_text);
+    }
+};
+
+fn path_exists(dir: std.fs.Dir, path: []const u8) !bool {
+    dir.access(path, .{}) catch |err| switch (err) {
+        error.FileNotFound => return false,
+        else => return err,
+    };
+    return true;
+}

data/ext/tb_client/tigerbeetle/src/docs_website/src/html.zig

@@ -0,0 +1,87 @@
+const std = @import("std");
+const log = std.log.scoped(.template);
+
+pub const Html = @This();
+
+arena: std.mem.Allocator,
+buffer: std.ArrayList(u8),
+writer: std.ArrayList(u8).Writer,
+
+pub fn create(arena: std.mem.Allocator) !*Html {
+    var html = try arena.create(Html);
+    html.* = .{
+        .arena = arena,
+        .buffer = std.ArrayList(u8).init(arena),
+        .writer = undefined,
+    };
+    html.writer = html.buffer.writer();
+    return html;
+}
+
+/// Replaces the variables in `$snake_case` matching the names of the fields in the
+/// `replacement` struct.
+///
+/// We picked the $dollar_name format over a {curly_braces} format to avoid potential ambiguities
+/// with JavaScript function syntax in the template.
+pub fn write(html: *Html, template: []const u8, replacements: anytype) !void {
+    const ReplacementsType = @TypeOf(replacements);
+    const replacements_type_info = @typeInfo(ReplacementsType);
+    if (replacements_type_info != .@"struct") @compileError("expected struct");
+
+    var unused = std.StringHashMap(void).init(html.arena);
+    inline for (replacements_type_info.@"struct".fields) |field| {
+        try unused.put(field.name, {});
+    }
+
+    var it = std.mem.tokenizeScalar(u8, template, '$');
+    if (template[0] != '$') if (it.next()) |prefix| try html.writer.writeAll(prefix);
+    while (it.next()) |chunk| {
+        const identifier_len = for (chunk, 0..) |c, index| {
+            switch (c) {
+                'a'...'z', '_' => {},
+                else => break index,
+            }
+        } else chunk.len;
+
+        const identifier = chunk[0..identifier_len];
+        const found = inline for (replacements_type_info.@"struct".fields) |field| {
+            if (std.mem.eql(u8, field.name, identifier)) {
+                try html.writer.writeAll(switch (field.type) {
+                    *Html => @field(replacements, field.name).string(),
+                    else => @field(replacements, field.name),
+                });
+                _ = unused.remove(field.name);
+                break true;
+            }
+        } else false;
+        if (!found) {
+            log.err("Html.write: identifier '{s}' not found in replacements", .{identifier});
+            return error.IdentifierNotFound;
+        }
+
+        try html.writer.writeAll(chunk[identifier_len..]);
+    }
+
+    if (unused.count() > 0) {
+        var unused_it = unused.keyIterator();
+        while (unused_it.next()) |unused_identifier| {
+            log.err("Html.write: identifier '{s}' not found in template", .{unused_identifier.*});
+        }
+        return error.UnusedIdentifiers;
+    }
+}
+
+pub fn child(self: Html) !*Html {
+    return try Html.create(self.arena);
+}
+
+pub fn string(self: Html) []const u8 {
+    return self.buffer.items;
+}
+
+pub fn redirect(arena: std.mem.Allocator, url: []const u8) ![]const u8 {
+    const template = @embedFile("html/redirect.html");
+    var html = try Html.create(arena);
+    try html.write(template, .{ .url = url });
+    return html.string();
+}