tigerbeetle 0.0.40 → 0.17.8

data/ext/tb_client/tigerbeetle/src/lsm/forest.zig

@@ -1,1146 +0,0 @@
-const std = @import("std");
-const builtin = @import("builtin");
-const assert = std.debug.assert;
-const maybe = stdx.maybe;
-const mem = std.mem;
-const log = std.log.scoped(.forest);
-
-const stdx = @import("stdx");
-const constants = @import("../constants.zig");
-
-const schema = @import("schema.zig");
-const GridType = @import("../vsr/grid.zig").GridType;
-const NodePool = @import("node_pool.zig").NodePoolType(constants.lsm_manifest_node_size, 16);
-const ManifestLogType = @import("manifest_log.zig").ManifestLogType;
-const ManifestLogPace = @import("manifest_log.zig").Pace;
-
-const ScratchMemory = @import("scratch_memory.zig").ScratchMemory;
-const ScanBufferPool = @import("scan_buffer.zig").ScanBufferPool;
-const ResourcePoolType = @import("compaction.zig").ResourcePoolType;
-const snapshot_min_for_table_output = @import("compaction.zig").snapshot_min_for_table_output;
-const compaction_op_min = @import("compaction.zig").compaction_op_min;
-const compaction_block_count_beat_min = @import("compaction.zig").compaction_block_count_beat_min;
-const compaction_input_tables_max = @import("compaction.zig").compaction_tables_input_max;
-
-/// The maximum number of tables for the forest as a whole. This is set a bit backwards due to how
-/// the code is structured: a single tree should be able to use all the tables in the forest, so the
-/// table_count_max of the forest is equal to the table_count_max of a single tree.
-/// In future, Forest.table_count_max could exceed Tree.table_count_max.
-pub const table_count_max = @import("tree.zig").table_count_max;
-
-pub fn ForestType(comptime _Storage: type, comptime groove_cfg: anytype) type {
-    const groove_count = std.meta.fields(@TypeOf(groove_cfg)).len;
-    var groove_fields: [groove_count]std.builtin.Type.StructField = undefined;
-    var groove_options_fields: [groove_count]std.builtin.Type.StructField = undefined;
-
-    for (std.meta.fields(@TypeOf(groove_cfg)), 0..) |field, i| {
-        const Groove = @field(groove_cfg, field.name);
-        groove_fields[i] = .{
-            .name = field.name,
-            .type = Groove,
-            .default_value_ptr = null,
-            .is_comptime = false,
-            .alignment = @alignOf(Groove),
-        };
-
-        groove_options_fields[i] = .{
-            .name = field.name,
-            .type = Groove.Options,
-            .default_value_ptr = null,
-            .is_comptime = false,
-            .alignment = @alignOf(Groove),
-        };
-    }
-
-    const _Grooves = @Type(.{
-        .@"struct" = .{
-            .layout = .auto,
-            .fields = &groove_fields,
-            .decls = &.{},
-            .is_tuple = false,
-        },
-    });
-
-    const _GroovesOptions = @Type(.{
-        .@"struct" = .{
-            .layout = .auto,
-            .fields = &groove_options_fields,
-            .decls = &.{},
-            .is_tuple = false,
-        },
-    });
-
-    {
-        // Verify that every tree id is unique.
-        comptime var ids: []const u16 = &.{};
-
-        inline for (std.meta.fields(_Grooves)) |groove_field| {
-            const Groove = groove_field.type;
-
-            for (std.meta.fields(@TypeOf(Groove.config.ids))) |field| {
-                const id = @field(Groove.config.ids, field.name);
-                assert(id > 0);
-                assert(std.mem.indexOfScalar(u16, ids, id) == null);
-
-                ids = ids ++ [_]u16{id};
-            }
-        }
-    }
-
-    const TreeInfo = struct {
-        Tree: type,
-        tree_name: []const u8,
-        tree_id: u16,
-        groove_name: []const u8,
-        groove_tree: union(enum) { objects, ids, indexes: []const u8 },
-    };
-
-    // Invariants:
-    // - tree_infos[tree_id - tree_id_range.min].tree_id == tree_id
-    // - tree_infos.len == tree_id_range.max - tree_id_range.min
-    const _tree_infos = tree_infos: {
-        @setEvalBranchQuota(32_000);
-
-        var tree_infos: []const TreeInfo = &[_]TreeInfo{};
-        for (std.meta.fields(_Grooves)) |groove_field| {
-            const Groove = groove_field.type;
-
-            tree_infos = tree_infos ++ &[_]TreeInfo{.{
-                .Tree = Groove.ObjectTree,
-                .tree_name = groove_field.name,
-                .tree_id = @field(Groove.config.ids, "timestamp"),
-                .groove_name = groove_field.name,
-                .groove_tree = .objects,
-            }};
-
-            if (Groove.IdTree != void) {
-                tree_infos = tree_infos ++ &[_]TreeInfo{.{
-                    .Tree = Groove.IdTree,
-                    .tree_name = groove_field.name ++ ".id",
-                    .tree_id = @field(Groove.config.ids, "id"),
-                    .groove_name = groove_field.name,
-                    .groove_tree = .ids,
-                }};
-            }
-
-            for (std.meta.fields(Groove.IndexTrees)) |tree_field| {
-                tree_infos = tree_infos ++ &[_]TreeInfo{.{
-                    .Tree = tree_field.type,
-                    .tree_name = groove_field.name ++ "." ++ tree_field.name,
-                    .tree_id = @field(Groove.config.ids, tree_field.name),
-                    .groove_name = groove_field.name,
-                    .groove_tree = .{ .indexes = tree_field.name },
-                }};
-            }
-        }
-
-        var tree_id_min = std.math.maxInt(u16);
-        for (tree_infos) |tree_info| tree_id_min = @min(tree_id_min, tree_info.tree_id);
-
-        var tree_infos_sorted: [tree_infos.len]TreeInfo = undefined;
-        var tree_infos_set: stdx.BitSetType(tree_infos.len) = .{};
-        for (tree_infos) |tree_info| {
-            const tree_index = tree_info.tree_id - tree_id_min;
-            assert(!tree_infos_set.is_set(tree_index));
-
-            tree_infos_sorted[tree_index] = tree_info;
-            tree_infos_set.set(tree_index);
-        }
-
-        // There are no gaps in the tree ids.
-        assert(tree_infos_set.full());
-
-        break :tree_infos tree_infos_sorted;
-    };
-
-    const _TreeID = comptime tree_id: {
-        var fields: [_tree_infos.len]std.builtin.Type.EnumField = undefined;
-        for (_tree_infos, 0..) |tree_info, i| {
-            fields[i] = .{
-                .name = @ptrCast(tree_info.tree_name),
-                .value = tree_info.tree_id,
-            };
-        }
-        break :tree_id @Type(.{ .@"enum" = .{
-            .tag_type = u16,
-            .fields = &fields,
-            .decls = &.{},
-            .is_exhaustive = true,
-        } });
-    };
-
-    comptime {
-        assert(std.enums.values(_TreeID).len == _tree_infos.len);
-        for (std.enums.values(_TreeID)) |tree_id| {
-            const tree_info = _tree_infos[@intFromEnum(tree_id) - _tree_infos[0].tree_id];
-            assert(tree_id == @as(_TreeID, @enumFromInt(tree_info.tree_id)));
-        }
-    }
-
-    const Grid = GridType(_Storage);
-
-    return struct {
-        const Forest = @This();
-
-        pub const ManifestLog = ManifestLogType(Storage);
-        const CompactionSchedule = CompactionScheduleType(Forest, Grid);
-
-        const Callback = *const fn (*Forest) void;
-
-        pub const Storage = _Storage;
-        pub const groove_config = groove_cfg;
-        pub const Grooves = _Grooves;
-        pub const GroovesOptions = _GroovesOptions;
-        // TreeID is an enum with a value for each tree type.
-        // Individual trees use `u16` to store their own id, to avoid dependency on the entire
-        // forest.
-        // Use `tree_id_cast` function to convert this type-erased u16 to a TreeID.
-        pub const TreeID = _TreeID;
-        pub const tree_infos = _tree_infos;
-        pub const tree_id_range = .{
-            .min = tree_infos[0].tree_id,
-            .max = tree_infos[tree_infos.len - 1].tree_id,
-        };
-
-        const manifest_log_compaction_pace = ManifestLogPace.init(.{
-            .tree_count = tree_infos.len,
-            // TODO Make this a runtime argument (from the CLI, derived from storage-size-max if
-            // possible).
-            .tables_max = table_count_max,
-            .compact_extra_blocks = constants.lsm_manifest_compact_extra_blocks,
-        });
-        pub const manifest_log_blocks_released_half_bar_max =
-            manifest_log_compaction_pace.half_bar_compact_blocks_max;
-
-        pub const Options = struct {
-            node_count: u32,
-            /// The amount of blocks allocated for compactions. Compactions will be deterministic
-            /// regardless of how much blocks you give them, but will run in fewer steps with more
-            /// memory.
-            compaction_block_count: u32,
-
-            pub const compaction_block_count_min: u32 = compaction_block_count_beat_min;
-        };
-
-        progress: ?union(enum) {
-            open: struct { callback: Callback },
-            checkpoint: struct { callback: Callback },
-            compact: struct {
-                op: u64,
-                callback: Callback,
-            },
-        } = null,
-
-        compaction_progress: ?struct {
-            trees_done: bool,
-            manifest_log_done: bool,
-
-            fn all_done(compaction_progress: @This()) bool {
-                return compaction_progress.trees_done and compaction_progress.manifest_log_done;
-            }
-        } = null,
-
-        grid: *Grid,
-        next_tick: Grid.NextTick = undefined,
-
-        grooves: Grooves,
-        node_pool: NodePool,
-        manifest_log: ManifestLog,
-
-        compaction_schedule: CompactionSchedule,
-
-        scan_buffer_pool: ScanBufferPool,
-
-        radix_buffer: ScratchMemory,
-
-        pub fn init(
-            forest: *Forest,
-            allocator: mem.Allocator,
-            grid: *Grid,
-            options: Options,
-            // (e.g.) .{ .transfers = .{ .cache_entries_max = 128, … }, .accounts = … }
-            grooves_options: GroovesOptions,
-        ) !void {
-            assert(options.compaction_block_count >= Options.compaction_block_count_min);
-            forest.* = .{
-                .grid = grid,
-                .grooves = undefined,
-                .node_pool = undefined,
-                .manifest_log = undefined,
-                .compaction_schedule = undefined,
-                .scan_buffer_pool = undefined,
-                .radix_buffer = undefined,
-            };
-
-            // TODO: look into using lsm_table_size_max for the node_count.
-            try forest.node_pool.init(allocator, options.node_count);
-            errdefer forest.node_pool.deinit(allocator);
-
-            try forest.manifest_log.init(
-                allocator,
-                grid,
-                &manifest_log_compaction_pace,
-            );
-            errdefer forest.manifest_log.deinit(allocator);
-
-            var grooves_initialized: usize = 0;
-            errdefer inline for (std.meta.fields(Grooves), 0..) |field, field_index| {
-                if (grooves_initialized >= field_index + 1) {
-                    const Groove = field.type;
-                    const groove: *Groove = &@field(forest.grooves, field.name);
-                    groove.deinit(allocator);
-                }
-            };
-
-            const radix_buffer_size: usize = comptime blk: {
-                var size_max: usize = 0;
-                for (std.enums.values(_TreeID)) |tree_id| {
-                    const tree = _tree_infos[@intFromEnum(tree_id) - _tree_infos[0].tree_id];
-                    const size = tree.Tree.Table.value_count_max * @sizeOf(tree.Tree.Value);
-                    assert(size > 0);
-                    size_max = @max(size_max, size);
-                }
-                break :blk size_max;
-            };
-
-            forest.radix_buffer = try .init(allocator, radix_buffer_size);
-            errdefer forest.radix_buffer.deinit(allocator);
-
-            inline for (std.meta.fields(Grooves)) |field| {
-                const Groove = field.type;
-                const groove: *Groove = &@field(forest.grooves, field.name);
-                const groove_options: Groove.Options = @field(grooves_options, field.name);
-
-                try groove.init(
-                    allocator,
-                    &forest.node_pool,
-                    grid,
-                    &forest.radix_buffer,
-                    groove_options,
-                );
-                grooves_initialized += 1;
-            }
-
-            try forest.compaction_schedule.init(
-                allocator,
-                grid,
-                forest,
-                options.compaction_block_count,
-            );
-            errdefer forest.compaction_schedule.deinit(allocator);
-
-            try forest.scan_buffer_pool.init(allocator);
-            errdefer forest.scan_buffer_pool.deinit(allocator);
-        }
-
-        pub fn deinit(forest: *Forest, allocator: mem.Allocator) void {
-            inline for (std.meta.fields(Grooves)) |field| {
-                const Groove = field.type;
-                const groove: *Groove = &@field(forest.grooves, field.name);
-                groove.deinit(allocator);
-            }
-
-            forest.manifest_log.deinit(allocator);
-            forest.node_pool.deinit(allocator);
-
-            forest.radix_buffer.deinit(allocator);
-
-            forest.compaction_schedule.deinit(allocator);
-            forest.scan_buffer_pool.deinit(allocator);
-        }
-
-        pub fn reset(forest: *Forest) void {
-            // Components using the node_pool must release all nodes they acquired upon reset.
-            defer assert(forest.node_pool.free.count() == forest.node_pool.free.bit_length);
-
-            inline for (std.meta.fields(Grooves)) |field| {
-                @field(forest.grooves, field.name).reset();
-            }
-
-            forest.grid.trace.cancel(.lookup);
-            forest.grid.trace.cancel(.lookup_worker);
-            forest.grid.trace.cancel(.scan_tree);
-            forest.grid.trace.cancel(.scan_tree_level);
-
-            forest.manifest_log.reset();
-            forest.scan_buffer_pool.reset();
-            forest.compaction_schedule.reset();
-
-            forest.* = .{
-                // Don't reset the grid – replica is responsible for grid cancellation.
-                .grid = forest.grid,
-                .grooves = forest.grooves,
-                .node_pool = forest.node_pool,
-                .manifest_log = forest.manifest_log,
-
-                .compaction_schedule = forest.compaction_schedule,
-
-                .scan_buffer_pool = forest.scan_buffer_pool,
-                .radix_buffer = forest.radix_buffer,
-            };
-        }
-
-        pub fn open(forest: *Forest, callback: Callback) void {
-            assert(forest.progress == null);
-            assert(forest.compaction_progress == null);
-
-            forest.progress = .{ .open = .{ .callback = callback } };
-
-            inline for (std.meta.fields(Grooves)) |field| {
-                @field(forest.grooves, field.name).open_commence(&forest.manifest_log);
-            }
-
-            forest.manifest_log.open(manifest_log_open_event, manifest_log_open_callback);
-        }
-
-        fn manifest_log_open_event(
-            manifest_log: *ManifestLog,
-            table: *const schema.ManifestNode.TableInfo,
-        ) void {
-            const forest: *Forest = @fieldParentPtr("manifest_log", manifest_log);
-            assert(forest.progress.? == .open);
-            assert(forest.compaction_progress == null);
-            assert(table.label.level < constants.lsm_levels);
-            assert(table.label.event != .remove);
-
-            if (table.tree_id < tree_id_range.min or table.tree_id > tree_id_range.max) {
-                log.err("manifest_log_open_event: unknown table in manifest: {}", .{table});
-                @panic("Forest.manifest_log_open_event: unknown table in manifest");
-            }
-            switch (tree_id_cast(table.tree_id)) {
-                inline else => |tree_id| {
-                    var tree: *TreeForIdType(tree_id) = forest.tree_for_id(tree_id);
-                    tree.open_table(table);
-                },
-            }
-        }
-
-        fn manifest_log_open_callback(manifest_log: *ManifestLog) void {
-            const forest: *Forest = @fieldParentPtr("manifest_log", manifest_log);
-            assert(forest.progress.? == .open);
-            assert(forest.compaction_progress == null);
-            forest.verify_tables_recovered();
-
-            inline for (std.meta.fields(Grooves)) |field| {
-                @field(forest.grooves, field.name).open_complete();
-            }
-            forest.verify_table_extents();
-
-            const callback = forest.progress.?.open.callback;
-            forest.progress = null;
-            callback(forest);
-        }
-
-        pub fn compact(forest: *Forest, callback: Callback, op: u64) void {
-            const compaction_beat = op % constants.lsm_compaction_ops;
-
-            const first_beat = compaction_beat == 0;
-            const last_half_beat = compaction_beat ==
-                @divExact(constants.lsm_compaction_ops, 2) - 1;
-            const half_beat = compaction_beat == @divExact(constants.lsm_compaction_ops, 2);
-            const last_beat = compaction_beat == constants.lsm_compaction_ops - 1;
-            assert(@as(usize, @intFromBool(first_beat)) + @intFromBool(last_half_beat) +
-                @intFromBool(half_beat) + @intFromBool(last_beat) <= 1);
-
-            log.debug("entering forest.compact() op={} constants.lsm_compaction_ops={} " ++
-                "first_beat={} last_half_beat={} half_beat={} last_beat={}", .{
-                op,
-                constants.lsm_compaction_ops,
-                first_beat,
-                last_half_beat,
-                half_beat,
-                last_beat,
-            });
-
-            assert(forest.progress == null);
-            forest.progress = .{ .compact = .{
-                .op = op,
-                .callback = callback,
-            } };
-
-            // Run trees and manifest log compaction in parallel, join in compact_finish.
-            assert(forest.compaction_progress == null);
-            forest.compaction_progress = .{
-                .trees_done = false,
-                .manifest_log_done = false,
-            };
-
-            // No compactions are run during the absolute first bar, or during
-            // the first bar of the checkpoint that we are currently recovering
-            // from (see `op_compacted` for the rationale behind this).
-            if (op < constants.lsm_compaction_ops or
-                forest.grid.superblock.working.vsr_state.op_compacted(op))
-            {
-                forest.compaction_progress.?.manifest_log_done = true;
-                forest.compaction_progress.?.trees_done = true;
-                forest.grid.on_next_tick(compact_finish_next_tick, &forest.next_tick);
-                return;
-            }
-
-            // Manifest log compaction. Run on the last beat of each half-bar.
-            // Start before forest compaction for lesser fragmentation, as
-            // manifest log grid reservations are much smaller than compaction's.
-            // TODO: Figure out a plan wrt the pacing here. Putting it on the
-            // last beat kinda-sorta balances out, because we expect to naturally
-            // do less other compaction work on the last beat.
-            if (last_beat or last_half_beat) {
-                forest.manifest_log.compact(compact_manifest_log_callback, op);
-            } else {
-                forest.compaction_progress.?.manifest_log_done = true;
-            }
-
-            forest.compaction_schedule.beat_start(compact_trees_callback, op);
-        }
-
-        fn compact_finish_next_tick(next_tick: *Grid.NextTick) void {
-            const forest: *Forest = @alignCast(
-                @fieldParentPtr("next_tick", next_tick),
-            );
-            forest.compact_finish();
-        }
-
-        fn compact_trees_callback(forest: *Forest) void {
-            assert(forest.progress.? == .compact);
-            assert(forest.compaction_progress != null);
-            assert(!forest.compaction_progress.?.trees_done);
-            forest.compaction_progress.?.trees_done = true;
-
-            if (forest.compaction_progress.?.all_done()) {
-                forest.compact_finish();
-            }
-        }
-
-        fn compact_manifest_log_callback(manifest_log: *ManifestLog) void {
-            const forest: *Forest = @fieldParentPtr("manifest_log", manifest_log);
-
-            assert(forest.progress.? == .compact);
-            assert(forest.compaction_progress != null);
-            assert(!forest.compaction_progress.?.manifest_log_done);
-            forest.compaction_progress.?.manifest_log_done = true;
-
-            if (forest.compaction_progress.?.all_done()) {
-                forest.compact_finish();
-            }
-        }
-
-        fn compact_finish(forest: *Forest) void {
-            assert(forest.progress.? == .compact);
-            assert(forest.compaction_progress != null);
-            assert(forest.compaction_progress.?.trees_done);
-            assert(forest.compaction_progress.?.manifest_log_done);
-            assert(forest.compaction_schedule.pool.idle());
-            assert(forest.compaction_schedule.pool.blocks_acquired() <=
-                compaction_block_count_beat_min);
-
-            forest.verify_table_extents();
-
-            assert(forest.progress.? == .compact);
-            const op = forest.progress.?.compact.op;
-
-            const compaction_beat = op % constants.lsm_compaction_ops;
-            const last_half_beat = compaction_beat ==
-                @divExact(constants.lsm_compaction_ops, 2) - 1;
-            const last_beat = compaction_beat == constants.lsm_compaction_ops - 1;
-
-            if (op < constants.lsm_compaction_ops or
-                forest.grid.superblock.working.vsr_state.op_compacted(op))
-            {
-                // No compaction was run.
-            } else {
-                for (0..constants.lsm_levels) |level_b| {
-                    if (level_active(.{ .level_b = level_b, .op = op })) {
-                        inline for (comptime std.enums.values(Forest.TreeID)) |tree_id| {
-                            const compaction =
-                                forest.compaction_schedule.compaction_at(level_b, tree_id);
-
-                            // Apply the changes to the manifest. This will run at the target
-                            // compaction beat that is requested.
-                            if (last_beat or last_half_beat) compaction.bar_complete();
-                        }
-                    }
-                }
-            }
-
-            // Groove sync compaction - must be done after all async work for the beat completes.
-            inline for (std.meta.fields(Grooves)) |field| {
-                @field(forest.grooves, field.name).compact(op);
-            }
-
-            if (last_beat or last_half_beat) {
-                if (forest.compaction_schedule.bar_input_size) |bar_input_size| {
-                    assert(bar_input_size == 0);
-                    forest.compaction_schedule.bar_input_size = null;
-                }
-
-                // On the last beat of the bar, make sure that manifest log compaction is finished.
-                forest.manifest_log.compact_end();
-
-                // Swap the mutable and immutable tables; this must happen on the last beat,
-                // regardless of pacing.
-                if (last_beat) {
-                    inline for (comptime std.enums.values(TreeID)) |tree_id| {
-                        const tree = tree_for_id(forest, tree_id);
-
-                        log.debug("swap_mutable_and_immutable({s})", .{tree.config.name});
-                        tree.swap_mutable_and_immutable(
-                            snapshot_min_for_table_output(compaction_op_min(op)),
-                        );
-
-                        // Ensure tables haven't overflowed.
-                        tree.manifest.assert_level_table_counts();
-                    }
-                }
-            }
-
-            const callback = forest.progress.?.compact.callback;
-            forest.progress = null;
-            forest.compaction_progress = null;
-
-            callback(forest);
-        }
-
-        pub fn checkpoint(forest: *Forest, callback: Callback) void {
-            assert(forest.progress == null);
-            assert(forest.compaction_progress == null);
-            forest.grid.assert_only_repairing();
-            forest.verify_table_extents();
-
-            forest.progress = .{ .checkpoint = .{ .callback = callback } };
-
-            inline for (std.meta.fields(Grooves)) |field| {
-                @field(forest.grooves, field.name).assert_between_bars();
-            }
-
-            inline for (comptime std.enums.values(TreeID)) |tree_id| {
-                const tree = tree_for_id(forest, tree_id);
-
-                // The last immutable table constructed before the checkpoint must not absorb any
-                // mutable table, because otherwise recovering from checkpoint would construct a
-                // different immutable table.
-                assert(!tree.table_immutable.mutability.immutable.absorbed);
-                maybe(tree.table_immutable.count() > 0);
-                assert(tree.table_mutable.count() == 0);
-            }
-
-            forest.manifest_log.checkpoint(checkpoint_manifest_log_callback);
-        }
-
-        fn checkpoint_manifest_log_callback(manifest_log: *ManifestLog) void {
-            const forest: *Forest = @fieldParentPtr("manifest_log", manifest_log);
-            assert(forest.progress.? == .checkpoint);
-            assert(forest.compaction_progress == null);
-            forest.verify_table_extents();
-            forest.verify_tables_recovered();
-
-            const callback = forest.progress.?.checkpoint.callback;
-            forest.progress = null;
-            callback(forest);
-        }
-
-        pub fn tree_id_cast(tree_id: u16) TreeID {
-            return @enumFromInt(tree_id);
-        }
-
-        fn TreeForIdType(comptime tree_id: TreeID) type {
-            const tree_info = tree_infos[@intFromEnum(tree_id) - tree_id_range.min];
-            assert(tree_info.tree_id == @intFromEnum(tree_id));
-
-            return tree_info.Tree;
-        }
-
-        pub fn tree_info_for_id(comptime tree_id: TreeID) TreeInfo {
-            const tree_info = tree_infos[@intFromEnum(tree_id) - tree_id_range.min];
-            assert(tree_info.tree_id == @intFromEnum(tree_id));
-
-            return tree_info;
-        }
-
-        pub fn tree_for_id(forest: *Forest, comptime tree_id: TreeID) *TreeForIdType(tree_id) {
-            const tree_info = tree_infos[@intFromEnum(tree_id) - tree_id_range.min];
-            assert(tree_info.tree_id == @intFromEnum(tree_id));
-
-            var groove = &@field(forest.grooves, tree_info.groove_name);
-
-            switch (tree_info.groove_tree) {
-                .objects => return &groove.objects,
-                .ids => return &groove.ids,
-                .indexes => |index_name| return &@field(groove.indexes, index_name),
-            }
-        }
-
-        pub fn tree_for_id_const(
-            forest: *const Forest,
-            comptime tree_id: TreeID,
-        ) *const TreeForIdType(tree_id) {
-            const tree_info = tree_infos[@intFromEnum(tree_id) - tree_id_range.min];
-            assert(tree_info.tree_id == @intFromEnum(tree_id));
-
-            const groove = &@field(forest.grooves, tree_info.groove_name);
-
-            switch (tree_info.groove_tree) {
-                .objects => return &groove.objects,
-                .ids => return &groove.ids,
-                .indexes => |index_name| return &@field(groove.indexes, index_name),
-            }
-        }
-
-        /// Returns whether the forest contains this table (ignoring differences in snapshot_max) at
-        /// any level.
-        pub fn contains_table(
-            forest: *const Forest,
-            table: *const schema.ManifestNode.TableInfo,
-        ) bool {
-            switch (tree_id_cast(table.tree_id)) {
-                inline else => |tree_id| {
-                    const tree = forest.tree_for_id_const(tree_id);
-                    const Tree = Forest.TreeForIdType(tree_id);
-                    const tree_table = Tree.Manifest.TreeTableInfo.decode(table);
-                    for (&tree.manifest.levels) |manifest_level| {
-                        if (manifest_level.find(&tree_table)) |level_table| {
-                            assert(tree_table.checksum == level_table.table_info.checksum);
-                            assert(tree_table.address == level_table.table_info.address);
-                            assert(tree_table.key_min == level_table.table_info.key_min);
-                            assert(tree_table.key_max == level_table.table_info.key_max);
-                            assert(tree_table.snapshot_min == level_table.table_info.snapshot_min);
-
-                            assert(tree_table.snapshot_max <= level_table.table_info.snapshot_max);
-                            return true;
-                        }
-                    }
-                    return false;
-                },
-            }
-        }
-
-        /// Verify that `ManifestLog.table_extents` has an extent for every active table.
-        ///
-        /// (Invoked between beats.)
-        fn verify_table_extents(forest: *const Forest) void {
-            var tables_count: usize = 0;
-            inline for (comptime std.enums.values(TreeID)) |tree_id| {
-                for (0..constants.lsm_levels) |level| {
-                    const tree_level = forest.tree_for_id_const(tree_id).manifest.levels[level];
-                    tables_count += tree_level.tables.len();
-
-                    if (constants.verify) {
-                        var tables_iterator = tree_level.tables.iterator_from_index(0, .ascending);
-                        while (tables_iterator.next()) |table| {
-                            assert(forest.manifest_log.table_extents.get(table.address) != null);
-                        }
-                    }
-                }
-            }
-            assert(tables_count == forest.manifest_log.table_extents.count());
-        }
-
-        /// Verify the tables recovered into the ManifestLevels after opening the manifest log.
-        ///
-        /// There are two strategies to reconstruct the LSM's manifest levels (i.e. the list of
-        /// tables) from a superblock manifest:
-        ///
-        /// 1. Iterate the manifest events in chronological order, replaying each
-        ///    insert/update/remove in sequence.
-        /// 2. Iterate the manifest events in reverse-chronological order, ignoring events for
-        ///    tables that have already been encountered.
-        ///
-        /// The manifest levels constructed by each strategy are identical.
-        ///
-        /// 1. This function implements strategy 1, to validate `ManifestLog.open()`.
-        /// 2. `ManifestLog.open()` implements strategy 2.
-        ///
-        /// (Strategy 2 minimizes the number of ManifestLevel mutations.)
-        ///
-        /// (Invoked immediately after open() or checkpoint()).
-        fn verify_tables_recovered(forest: *const Forest) void {
-            const ForestTableIteratorType =
-                @import("./forest_table_iterator.zig").ForestTableIteratorType;
-            const ForestTableIterator = ForestTableIteratorType(Forest);
-
-            assert(forest.grid.superblock.opened);
-            assert(forest.manifest_log.opened);
-
-            if (Forest.Storage != @import("../testing/storage.zig").Storage) return;
-
-            // The manifest log is opened, which means we have all of the manifest blocks.
-            // But if the replica is syncing, those blocks might still be writing (and thus not in
-            // the TestStorage when we go to retrieve them).
-            if (forest.grid.superblock.working.vsr_state.sync_op_max > 0) return;
-
-            // The latest version of each table, keyed by table checksum.
-            // Null when the table has been deleted.
-            var tables_latest = std.AutoHashMap(u128, struct {
-                table: schema.ManifestNode.TableInfo,
-                manifest_block: u64,
-                manifest_entry: u32,
-            }).init(forest.grid.superblock.storage.allocator);
-            defer tables_latest.deinit();
-
-            // Replay manifest events in chronological order.
-            // Accumulate all tables that belong in the recovered forest's ManifestLevels.
-            for (0..forest.manifest_log.log_block_checksums.count) |i| {
-                const block_checksum = forest.manifest_log.log_block_checksums.get(i).?;
-                const block_address = forest.manifest_log.log_block_addresses.get(i).?;
-                assert(block_address > 0);
-
-                const block = forest.grid.superblock.storage.grid_block(block_address).?;
-                const block_header = schema.header_from_block(block);
-                assert(block_header.address == block_address);
-                assert(block_header.checksum == block_checksum);
-                assert(block_header.block_type == .manifest);
-
-                const block_schema = schema.ManifestNode.from(block);
-                assert(block_schema.entry_count > 0);
-                assert(block_schema.entry_count <= schema.ManifestNode.entry_count_max);
-
-                for (block_schema.tables_const(block), 0..) |*table, entry| {
-                    if (table.label.event == .remove) {
-                        maybe(tables_latest.remove(table.checksum));
-                    } else {
-                        tables_latest.put(table.checksum, .{
-                            .table = table.*,
-                            .manifest_block = block_address,
-                            .manifest_entry = @intCast(entry),
-                        }) catch @panic("oom");
-                    }
-                }
-
-                if (i > 0) {
-                    // Verify the linked-list.
-                    const block_previous = schema.ManifestNode.previous(block).?;
-                    assert(block_previous.checksum ==
-                        forest.manifest_log.log_block_checksums.get(i - 1).?);
-                    assert(block_previous.address ==
-                        forest.manifest_log.log_block_addresses.get(i - 1).?);
-                }
-            }
-
-            // Verify that the SuperBlock Manifest's table extents are correct.
-            var tables_latest_iterator = tables_latest.valueIterator();
-            var table_extent_counts: usize = 0;
-            while (tables_latest_iterator.next()) |table| {
-                const table_extent = forest.manifest_log.table_extents.get(table.table.address).?;
-                assert(table.manifest_block == table_extent.block);
-                assert(table.manifest_entry == table_extent.entry);
-
-                table_extent_counts += 1;
-            }
-            assert(table_extent_counts == forest.manifest_log.table_extents.count());
-
-            // Verify the tables in `tables` are exactly the tables recovered by the Forest.
-            var forest_tables_iterator = ForestTableIterator{};
-            while (forest_tables_iterator.next(forest)) |forest_table_item| {
-                const table_latest = tables_latest.get(forest_table_item.checksum).?;
-                assert(table_latest.table.label.level == forest_table_item.label.level);
-                assert(std.meta.eql(table_latest.table.key_min, forest_table_item.key_min));
-                assert(std.meta.eql(table_latest.table.key_max, forest_table_item.key_max));
-                assert(table_latest.table.checksum == forest_table_item.checksum);
-                assert(table_latest.table.address == forest_table_item.address);
-                assert(table_latest.table.snapshot_min == forest_table_item.snapshot_min);
-                assert(table_latest.table.snapshot_max == forest_table_item.snapshot_max);
-                assert(table_latest.table.tree_id == forest_table_item.tree_id);
-
-                const table_removed = tables_latest.remove(forest_table_item.checksum);
-                assert(table_removed);
-            }
-            assert(tables_latest.count() == 0);
-        }
-
-        /// Calculates the maximum number of blocks that could be released by Tree and ManifestLog
-        /// compactions before a checkpoint becomes durable on a commit quorum of replicas.
-        ///
-        /// A checkpoint is guaranteed to be durable when a replica commits the (pipeline + 1)th
-        /// prepare after checkpoint trigger (see `op_repair_min` in replica.zig for more details).
-        /// Therefore, the maximum number of blocks released prior checkpoint durability is
-        /// equivalent to the maximum number of blocks released by the first pipeline of prepares
-        /// after checkpoint trigger.
-        pub fn compaction_blocks_released_per_pipeline_max() usize {
-            const half_bar_ops = @divExact(constants.lsm_compaction_ops, 2);
-            const pipeline_half_bars =
-                stdx.div_ceil(constants.pipeline_prepare_queue_max, half_bar_ops);
-
-            // Maximum number of blocks released within a single half-bar by compaction.
-            const compaction_blocks_released_half_bar_max = blocks: {
-                var blocks: usize = 0;
-                inline for (Forest.tree_infos) |tree_info| {
-                    blocks +=
-                        stdx.div_ceil(constants.lsm_levels, 2) *
-                        (compaction_input_tables_max *
-                            (1 + tree_info.Tree.Table.layout.value_block_count_max));
-                }
-                break :blocks blocks;
-            };
-
-            const compaction_blocks_released_pipeline_max =
-                (pipeline_half_bars * compaction_blocks_released_half_bar_max) +
-                // Compaction is paced across all beats, so if a pipeline is less than half a bar,
-                // for simplicity, use the upper bound for a half a bar (treating pacing as
-                // imperfect).
-                @intFromBool(pipeline_half_bars == 0) * compaction_blocks_released_half_bar_max;
-
-            // Maximum number of blocks released within a pipeline by ManifestLog compactions.
-            const manifest_log_blocks_released_pipeline_max =
-                pipeline_half_bars * Forest.manifest_log_blocks_released_half_bar_max;
-
-            return compaction_blocks_released_pipeline_max +
-                manifest_log_blocks_released_pipeline_max;
-        }
-    };
-}
-
-/// Plans a bar's worth of compaction work across all the trees in the Forest, and schedules it
-/// one beat at a time. Each bar is divided into two half bars with `lsm_compaction_ops/2` beats
-/// each. Even levels (0 → 1, 2 → 4, etc.) are active during the first half bar and odd levels
-/// (immutable → 0, 1 → 3, etc.) are active during the second half bar.
-///
-/// We now describe the scheduling algorithm. In the description, we refer to each (tree, level)
-/// combination as a `Compaction`, for example the compaction from level 0 → 1 in the Accounts tree.
-///
-/// At the first beat of each half bar:
-/// 1. Calculate the half-bar quota for each Compaction, which is the total number of bytes that
-///    Compaction needs to chew through. We use this as an estimate of time the compaction will take
-///    and then slice it into small chunks, to spread it evenly across the beats of the half bar.
-/// 2. Calculate the half-bar quota for the entire Forest by summing up the aforementioned quotas.
-///
-/// At each beat:
-/// 1. Calculate the Forest's beat quota by equally dividing the half-bar quota across each beat.
-/// 2. Resume a suspended Compaction, or start a new one.
-/// 3. Run active Compaction till either the Forest's beat quota is met, or its half-bar quota is
-///    met. If its the latter, go to step 2. If its the former...
-/// 4. Suspend active Compaction and finish the beat.
-fn CompactionScheduleType(comptime Forest: type, comptime Grid: type) type {
-    return struct {
-        grid: *Grid,
-        forest: *Forest,
-        pool: ResourcePool,
-        next_tick: Grid.NextTick = undefined,
-        callback: ?*const fn (*Forest) void = null,
-        bar_input_size: ?u64 = null,
-        beat_input_size: ?u64 = null,
-
-        const CompactionSchedule = @This();
-        const ResourcePool = ResourcePoolType(Grid);
-
-        pub fn init(
-            self: *CompactionSchedule,
-            allocator: mem.Allocator,
-            grid: *Grid,
-            forest: *Forest,
-            block_count: u32,
-        ) !void {
-            assert(block_count >= compaction_block_count_beat_min);
-
-            self.* = .{ .grid = grid, .forest = forest, .pool = undefined };
-            self.pool = try ResourcePool.init(allocator, block_count);
-            errdefer self.pool.deinit(allocator);
-        }
-
-        pub fn deinit(self: *CompactionSchedule, allocator: mem.Allocator) void {
-            self.pool.deinit(allocator);
-        }
-
-        pub fn reset(self: *CompactionSchedule) void {
-            self.pool.reset();
-
-            self.* = .{ .grid = self.grid, .forest = self.forest, .pool = self.pool };
-        }
-
-        pub fn beat_start(self: *CompactionSchedule, callback: Forest.Callback, op: u64) void {
-            assert(self.pool.idle());
-            assert(self.pool.grid_reservation == null);
-
-            assert(self.callback == null);
-            assert(op >= constants.lsm_compaction_ops);
-            assert(!self.grid.superblock.working.vsr_state.op_compacted(op));
-            assert(self.beat_input_size == null);
-
-            self.beat_input_size = 0;
-            self.callback = callback;
-
-            const half_bar = @divExact(constants.lsm_compaction_ops, 2);
-            const compaction_beat = op % constants.lsm_compaction_ops;
-
-            const first_beat = compaction_beat == 0;
-            const half_beat = compaction_beat == half_bar;
-
-            if (first_beat or half_beat) {
-                assert(self.pool.blocks_acquired() == 0);
-                assert(self.bar_input_size == null);
-
-                var bar_input_size: u64 = 0;
-                for (0..constants.lsm_levels) |level_b| {
-                    if (level_active(.{ .level_b = level_b, .op = op })) {
-                        inline for (comptime std.enums.values(Forest.TreeID)) |tree_id| {
-                            const tree = Forest.tree_info_for_id(tree_id);
-                            const Value = tree.Tree.Value;
-                            const compaction = self.compaction_at(level_b, tree_id);
-
-                            assert(
-                                self.pool.blocks_free() >=
-                                    // Input index & value blocks may be carried to the next beat.
-                                    compaction.level_a_index_block.buffer.len +
-                                        compaction.level_a_value_block.buffer.len +
-                                        compaction.level_b_index_block.buffer.len +
-                                        compaction.level_b_value_block.buffer.len +
-                                        // At least one output index & value block.
-                                        (1 + 1),
-                            );
-                            const bar_input_values = compaction.bar_commence(op);
-
-                            bar_input_size += (bar_input_values * @sizeOf(Value));
-                        }
-                    }
-                }
-                self.bar_input_size = bar_input_size;
-            }
-
-            const beats_total = half_bar;
-            const beats_done = compaction_beat % half_bar;
-            const beats_remaining = beats_total - beats_done;
-
-            self.beat_input_size = stdx.div_ceil(self.bar_input_size.?, beats_remaining);
-
-            // This is akin to a dry run for the actual compaction work that is going to happen
-            // during this beat, wherein we:
-            // * Invoke beat_commence on the active compactions to set beat quotas
-            // * Reserve blocks in the grid for the output of these compactions
-            {
-                // 1 since we may have partially finished index/value blocks from the previous beat.
-                var beat_index_blocks_max: u64 = 1;
-                var beat_value_blocks_max: u64 = 1;
-
-                var beat_input_size = self.beat_input_size.?;
-                for (0..constants.lsm_levels) |level_b| {
-                    if (level_active(.{ .level_b = level_b, .op = op })) {
-                        inline for (comptime std.enums.values(Forest.TreeID)) |tree_id| {
-                            const tree = Forest.tree_info_for_id(tree_id);
-                            const compaction = self.compaction_at(level_b, tree_id);
-
-                            const Value = tree.Tree.Value;
-                            const Table = tree.Tree.Table;
-
-                            compaction.beat_commence(
-                                stdx.div_ceil(beat_input_size, @sizeOf(Value)),
-                            );
-
-                            // The +1 is for imperfections in pacing our immutable table, which
-                            // might cause us to overshoot by a single block (limited to 1 due
-                            // to how the immutable table values are consumed.)
-                            const beat_value_blocks = stdx.div_ceil(
-                                compaction.quotas.beat,
-                                Table.layout.block_value_count_max,
-                            ) + 1;
-                            const beat_index_blocks = stdx.div_ceil(
-                                beat_value_blocks,
-                                Table.value_block_count_max,
-                            );
-                            beat_value_blocks_max += beat_value_blocks;
-                            beat_index_blocks_max += beat_index_blocks;
-
-                            beat_input_size -|= (compaction.quotas.beat * @sizeOf(Value));
-                        }
-                    }
-                }
-                assert(beat_input_size == 0);
-                self.pool.grid_reservation = self.grid.reserve(
-                    beat_value_blocks_max + beat_index_blocks_max,
-                );
-            }
-
-            self.beat_resume();
-        }
-
-        fn beat_resume(self: *CompactionSchedule) void {
-            assert(self.callback != null);
-            assert(self.pool.grid_reservation != null);
-
-            if (self.beat_input_size == 0) {
-                self.beat_finish();
-                return;
-            }
-
-            const op = self.forest.progress.?.compact.op;
-
-            for (0..constants.lsm_levels) |level_b| {
-                if (level_active(.{ .level_b = level_b, .op = op })) {
-                    inline for (comptime std.enums.values(Forest.TreeID)) |tree_id| {
-                        const compaction = self.compaction_at(level_b, tree_id);
-
-                        const resumed = compaction.compaction_dispatch_enter(.{
-                            .pool = &self.pool,
-                            .callback = beat_resume_callback,
-                        });
-
-                        switch (resumed) {
-                            .pending => return,
-                            .ready => {},
-                        }
-                    }
-                }
-            }
-        }
-
-        fn beat_resume_callback(pool: *ResourcePool, tree_id: u16, values_consumed: u64) void {
-            const self: *CompactionSchedule = @fieldParentPtr("pool", pool);
-            assert(self.callback != null);
-
-            switch (Forest.tree_id_cast(tree_id)) {
-                inline else => |id| {
-                    const Value = Forest.tree_info_for_id(id).Tree.Value;
-                    const input_bytes_consumed = values_consumed * @sizeOf(Value);
-                    self.bar_input_size.? -= input_bytes_consumed;
-                    self.beat_input_size.? -|= input_bytes_consumed;
-                },
-            }
-
-            self.beat_resume();
-        }
-
-        fn beat_finish(self: *CompactionSchedule) void {
-            assert(self.callback != null);
-
-            assert(self.bar_input_size.? >= 0);
-            assert(self.beat_input_size == 0);
-            self.beat_input_size = null;
-
-            if (self.pool.grid_reservation) |reservation| {
-                self.grid.forfeit(reservation);
-                self.pool.grid_reservation = null;
-            }
-
-            self.grid.on_next_tick(beat_finish_next_tick, &self.next_tick);
-        }
-
-        fn beat_finish_next_tick(next_tick: *Grid.NextTick) void {
-            const self: *CompactionSchedule = @alignCast(
-                @fieldParentPtr("next_tick", next_tick),
-            );
-            const callback = self.callback.?;
-            self.callback = null;
-            callback(self.forest);
-        }
-
-        fn compaction_at(
-            self: *CompactionSchedule,
-            level_b: usize,
-            comptime tree_id: Forest.TreeID,
-        ) *Forest.TreeForIdType(tree_id).Compaction {
-            return &self.forest.tree_for_id(tree_id).compactions[level_b];
-        }
-    };
-}
-
-fn level_active(options: struct { level_b: usize, op: u64 }) bool {
-    const half_bar_beat_count = @divExact(constants.lsm_compaction_ops, 2);
-    const compaction_beat = options.op % constants.lsm_compaction_ops;
-    return (compaction_beat < half_bar_beat_count) == (options.level_b % 2 == 1);
-}
-
-test level_active {
-    assert(!level_active(.{ .level_b = 0, .op = constants.lsm_compaction_ops }));
-    assert(level_active(.{ .level_b = 1, .op = constants.lsm_compaction_ops }));
-    assert(!level_active(.{ .level_b = 2, .op = constants.lsm_compaction_ops }));
-
-    assert(level_active(.{ .level_b = 0, .op = @divExact(constants.lsm_compaction_ops, 2) }));
-    assert(!level_active(.{ .level_b = 1, .op = @divExact(constants.lsm_compaction_ops, 2) }));
-    assert(level_active(.{ .level_b = 2, .op = @divExact(constants.lsm_compaction_ops, 2) }));
-}