tigerbeetle 0.0.34 → 0.0.37

data/ext/tb_client/tigerbeetle/src/lsm/compaction.zig

@@ -0,0 +1,2117 @@
+//! Compaction moves or merges a table's values from the previous level.
+//!
+//! Each Compaction is paced to run in an arbitrary amount of beats, by the forest.
+//!
+//! Compaction overview:
+//!
+//! 1. Given:
+//!
+//!   - levels A and B, where A+1=B
+//!   - a single table in level A ("table A")
+//!   - all tables from level B which intersect table A's key range ("tables B")
+//!     (This can include anything between 0 tables and all of level B's tables.)
+//!
+//! 2. If table A's key range is disjoint from the keys in level B, move table A into level B.
+//!    All done! (But if the key ranges intersect, jump to step 3).
+//!
+//! 3. Create an iterator from the sort-merge of table A and the concatenation of tables B.
+//!    If the same key exists in level A and B, take A's and discard B's. †
+//!
+//! 4. Write the sort-merge iterator into a sequence of new tables on disk.
+//!
+//! 5. Update the input tables in the Manifest with their new `snapshot_max` so that they become
+//!    invisible to subsequent read transactions.
+//!
+//! 6. Insert the new level-B tables into the Manifest.
+//!
+//! † When A's value is a tombstone, there is a special case for garbage collection. When either:
+//! * level B is the final level, or
+//! * A's key does not exist in B or any deeper level,
+//! then the tombstone is omitted from the compacted output, see: `compaction_must_drop_tombstones`.
+//!
+const std = @import("std");
+const mem = std.mem;
+const math = std.math;
+const assert = std.debug.assert;
+const Allocator = std.mem.Allocator;
+
+const log = std.log.scoped(.compaction);
+
+const constants = @import("../constants.zig");
+
+const stdx = @import("stdx");
+const maybe = stdx.maybe;
+const vsr = @import("../vsr.zig");
+const trace = @import("../trace.zig");
+const StackType = @import("../stack.zig").StackType;
+const IOPSType = @import("../iops.zig").IOPSType;
+const GridType = @import("../vsr/grid.zig").GridType;
+const BlockPtr = @import("../vsr/grid.zig").BlockPtr;
+const BlockPtrConst = @import("../vsr/grid.zig").BlockPtrConst;
+const allocate_block = @import("../vsr/grid.zig").allocate_block;
+const TableInfoType = @import("manifest.zig").TreeTableInfoType;
+const ManifestType = @import("manifest.zig").ManifestType;
+const schema = @import("schema.zig");
+const RingBufferType = stdx.RingBufferType;
+
+/// The upper-bound count of input tables to a single tree's compaction.
+///
+/// - +1 from level A.
+/// - +lsm_growth_factor from level B. The A-input table cannot overlap with an extra B-input table
+///   because input table selection is least-overlap. If the input table overlaps on one or both
+///   edges, there must be another table with less overlap to select.
+pub const compaction_tables_input_max = 1 + constants.lsm_growth_factor;
+
+/// The upper-bound count of output tables from a single tree's compaction.
+/// In the "worst" case, no keys are overwritten/merged, and no tombstones are dropped.
+pub const compaction_tables_output_max = compaction_tables_input_max;
+
+/// The minimum number of blocks required for a single beat of a single compaction.
+///
+/// Compaction needs to carry over the output index block and all input blocks to the next beat:
+/// One index and one value block for the output table, one index block for level A, two index
+/// blocks for level B (to allow prefetching), and `lsm_compaction_queue_read_max/2` value blocks
+/// for the two input tables.
+pub const compaction_block_count_beat_min: u32 =
+    (1 + 1) + (1 + 2) + constants.lsm_compaction_queue_read_max;
+
+const half_bar_beat_count = @divExact(constants.lsm_compaction_ops, 2);
+
+/// Resources shared by all compactions.
+///
+/// ResourcePool is a singleton owned by the Forest, but it doesn't depend on Forest type.
+pub fn ResourcePoolType(comptime Grid: type) type {
+    return struct {
+        reads: IOPSType(BlockRead, constants.lsm_compaction_iops_read_max) = .{},
+        writes: IOPSType(BlockWrite, constants.lsm_compaction_iops_write_max) = .{},
+        cpus: IOPSType(CPU, 1) = .{},
+        blocks: StackType(Block),
+        blocks_backing_storage: []Block,
+        grid_reservation: ?Grid.Reservation = null,
+
+        const ResourcePool = @This();
+
+        const BlockRead = struct {
+            grid_read: Grid.Read,
+            block: *Block,
+            compaction: *anyopaque,
+
+            fn parent(read: *BlockRead, comptime Compaction: type) *Compaction {
+                return @as(*Compaction, @ptrCast(@alignCast(read.compaction)));
+            }
+        };
+
+        const BlockWrite = struct {
+            grid_write: Grid.Write,
+            block: *Block,
+            compaction: *anyopaque,
+
+            fn parent(write: *BlockWrite, comptime Compaction: type) *Compaction {
+                return @as(*Compaction, @ptrCast(@alignCast(write.compaction)));
+            }
+        };
+
+        /// While we don't currently have a CPU pool, we already treat CPU as a resource, by storing
+        /// it in a ring-buffer of length one.
+        const CPU = struct {
+            next_tick: Grid.NextTick,
+            block: *Block,
+            compaction: *anyopaque,
+
+            fn parent(cpu: *CPU, comptime Compaction: type) *Compaction {
+                return @as(*Compaction, @ptrCast(@alignCast(cpu.compaction)));
+            }
+        };
+
+        const Block = struct {
+            ptr: BlockPtr,
+            stage: enum {
+                // block is in the resource pool.
+                free,
+
+                // Block is owned by a table builder.
+                build_index_block,
+                build_value_block,
+
+                // Block is in the read queue.
+                read_index_block,
+                read_index_block_done,
+                read_value_block,
+                read_value_block_done,
+
+                // Block is in the read queue and is used by merge.
+                // Next stage is either free or loops back to read_value_block_done.
+                merge,
+
+                // Block is in the write queue. Goes directly to free from here.
+                write_value_block,
+                write_index_block,
+            },
+
+            link: StackType(Block).Link,
+        };
+
+        pub fn init(allocator: mem.Allocator, block_count: u32) !ResourcePool {
+            const blocks_backing_storage = try allocator.alloc(Block, block_count);
+            var blocks_allocated: u32 = 0;
+            errdefer {
+                for (blocks_backing_storage[0..blocks_allocated]) |block| {
+                    allocator.free(block.ptr);
+                }
+                allocator.free(blocks_backing_storage);
+            }
+
+            for (blocks_backing_storage) |*block| {
+                block.* = .{
+                    .ptr = try allocate_block(allocator),
+                    .stage = .free,
+                    .link = .{},
+                };
+                blocks_allocated += 1;
+            }
+            assert(blocks_allocated == block_count);
+
+            var blocks = StackType(Block).init(.{
+                .capacity = blocks_allocated,
+                .verify_push = false,
+            });
+            for (blocks_backing_storage) |*block| blocks.push(block);
+
+            return .{
+                .blocks = blocks,
+                .blocks_backing_storage = blocks_backing_storage,
+            };
+        }
+
+        pub fn deinit(pool: *ResourcePool, allocator: Allocator) void {
+            for (pool.blocks_backing_storage) |block| {
+                allocator.free(block.ptr);
+            }
+            allocator.free(pool.blocks_backing_storage);
+        }
+
+        pub fn reset(pool: *ResourcePool) void {
+            pool.* = .{
+                .blocks = StackType(Block).init(.{
+                    .capacity = pool.blocks.capacity(),
+                    .verify_push = false,
+                }),
+                .blocks_backing_storage = pool.blocks_backing_storage,
+            };
+            for (pool.blocks_backing_storage) |*block| {
+                block.* = .{
+                    .ptr = block.ptr,
+                    .stage = .free,
+                    .link = .{},
+                };
+                pool.blocks.push(block);
+            }
+        }
+
+        // NB: idle does not check that no blocks are acquired! Although no IO can happen between
+        // the beats, it is valid to carry over some blocks.
+        pub fn idle(pool: *ResourcePool) bool {
+            return pool.reads.executing() == 0 and
+                pool.writes.executing() == 0 and
+                pool.cpus.executing() == 0;
+        }
+
+        pub fn blocks_acquired(pool: *ResourcePool) u32 {
+            assert(pool.blocks.count() <= pool.blocks_backing_storage.len);
+            return @as(u32, @intCast(pool.blocks_backing_storage.len - pool.blocks.count()));
+        }
+
+        pub fn blocks_free(pool: *ResourcePool) u32 {
+            return pool.blocks.count();
+        }
+
+        fn block_acquire(pool: *@This()) ?*Block {
+            const block = pool.blocks.pop() orelse return null;
+            assert(block.stage == .free);
+            assert(block.link.next == null);
+            return block;
+        }
+
+        fn block_release(pool: *@This(), block: *Block) void {
+            assert(block.stage == .free);
+            assert(block.link.next == null);
+            pool.blocks.push(block);
+        }
+
+        pub fn format(
+            self: @This(),
+            comptime _: []const u8,
+            _: std.fmt.FormatOptions,
+            writer: anytype,
+        ) !void {
+            return writer.print("ResourcePool{{ " ++
+                ".reads = {}/{},  .writes = {}/{}, .cpus = {}/{}, .blocks = {}/{} }}", .{
+                self.reads.available(),  self.reads.total(),
+                self.writes.available(), self.writes.total(),
+                self.cpus.available(),   self.cpus.total(),
+                self.blocks.count,       self.blocks_backing_storage.len,
+            });
+        }
+    };
+}
+
+pub fn CompactionType(
+    comptime Table: type,
+    comptime Tree: type,
+    comptime Storage: type,
+) type {
+    return struct {
+        const Compaction = @This();
+
+        const Grid = GridType(Storage);
+        const ResourcePool = ResourcePoolType(Grid);
+
+        const Manifest = ManifestType(Table, Storage);
+        const TableInfo = TableInfoType(Table);
+        const TableInfoReference = Manifest.TableInfoReference;
+        const CompactionRange = Manifest.CompactionRange;
+
+        const Value = Table.Value;
+        const key_from_value = Table.key_from_value;
+        const tombstone = Table.tombstone;
+
+        const TableInfoA = union(enum) {
+            immutable: []Value,
+            disk: TableInfoReference,
+        };
+
+        const Position = struct {
+            index_block: u32 = 0,
+            value_block: u32 = 0,
+            value: u32 = 0,
+
+            pub fn format(
+                self: @This(),
+                comptime _: []const u8,
+                _: std.fmt.FormatOptions,
+                writer: anytype,
+            ) !void {
+                return writer.print("Position{{ .index_block = {}, " ++
+                    ".value_block = {}, .value = {} }}", .{
+                    self.index_block,
+                    self.value_block,
+                    self.value,
+                });
+            }
+        };
+
+        // Globally scoped fields:
+        // ----------------------
+        grid: *Grid,
+        tree: *Tree,
+        level_b: u8,
+
+        stage: enum {
+            inactive,
+            beat,
+            beat_quota_done,
+            paused,
+        } = .inactive,
+
+        // Bar-scoped fields:
+        // -----------------
+
+        /// `op_min` is the first op/beat of this compaction's half-bar.
+        /// `op_min` is used as a snapshot — the compaction's input tables must be visible
+        /// to `op_min`.
+        ///
+        /// After this compaction finishes:
+        /// - `op_min + half_bar_beat_count - 1` will be the input tables' snapshot_max.
+        /// - `op_min + half_bar_beat_count` will be the output tables' snapshot_min.
+        op_min: u64 = 0,
+
+        table_info_a: ?TableInfoA = null,
+        range_b: ?CompactionRange = null,
+
+        /// Whether this compaction will use the move-table optimization.
+        /// Specifically, this field is set to True if the optimal compaction
+        /// table in level A can simply be moved to level B.
+        move_table: bool = false,
+        /// Levels may choose to drop tombstones if keys aren't included in the lower levels.
+        /// This invariant is always true for the last level as it doesn't have any lower ones.
+        drop_tombstones: bool = false,
+
+        /// Counters track physical IO and are not fully deterministic. In particular, `in` and
+        /// `dropped` values can vary between the replicas.
+        ///
+        /// Counters obey accounting equation of compaction:
+        ///     out = in - dropped
+        counters: struct {
+            in: u64 = 0,
+            dropped: u64 = 0, // Tombstones.
+            out: u64 = 0,
+
+            fn consistent(counters: @This()) bool {
+                return counters.out == counters.in - counters.dropped;
+            }
+        } = .{},
+
+        /// Quotas track logical progress of compaction, determine pacing and must be deterministic.
+        /// Quotas count consumed input values. That is, every time an output block is written,
+        /// the done quota is incremented by the number of input values which contributed to the
+        /// output block.
+        ///
+        /// At the start of the bar, the total number of input values is known. The beat quota is
+        /// then set based on the number of values left and beats left.
+        quotas: struct {
+            beat: u64 = 0,
+            beat_done: u64 = 0,
+            bar: u64 = 0,
+            bar_done: u64 = 0,
+
+            fn beat_exhausted(quotas: @This()) bool {
+                assert(quotas.beat_done <= quotas.bar_done);
+                assert(quotas.bar_done <= quotas.bar);
+                return quotas.beat_done >= quotas.beat;
+            }
+
+            fn bar_exhausted(quotas: @This()) bool {
+                assert(quotas.bar_done <= quotas.bar);
+                return quotas.bar_done == quotas.bar;
+            }
+        } = .{},
+
+        // Position points at the next value from the layer that should be feed into the merge
+        // algorithm.
+        level_a_position: Position = .{},
+        level_b_position: Position = .{},
+
+        /// Manifest log appends are queued up until bar_complete is explicitly called to ensure
+        /// they are applied deterministically relative to other concurrent compactions.
+        // Worst-case manifest updates:
+        // See docs/about/internals/lsm.md "Compaction Table Overlap" for more detail.
+        manifest_entries: stdx.BoundedArrayType(struct {
+            operation: enum {
+                insert_to_level_b,
+                move_to_level_b,
+            },
+            table: TableInfo,
+        }, compaction_tables_output_max) = .{},
+
+        table_builder: Table.Builder = .{},
+        table_builder_index_block: ?*ResourcePool.Block = null,
+        table_builder_value_block: ?*ResourcePool.Block = null,
+
+        // The progress through immutable table is persisted throughout the bar.
+        level_a_immutable_stage: enum { ready, merge, exhausted } = .ready,
+
+        // Beat-scoped fields:
+        // ------------------
+        pool: ?*ResourcePool = null,
+        callback: ?*const fn (pool: *ResourcePool, tree_id: u16, values_consumed: u64) void = null,
+
+        // IO queues:
+        //
+        // Compaction structure is such that the data can be read (and written) concurrently, but
+        // the merge must happen sequentially. It is reminiscent of state machine's prefetch/execute
+        // split.
+        //
+        // When a block is read from disk, it is added to the tail of the corresponding queue. When
+        // the head block from both level a and level b queues is in the .read_value_block_done
+        // state, the two blocks are popped off the queues and passed down to the merge. At this
+        // point, any number of the blocks still in the queues can continue their read operations.
+        //
+        // For index blocks, queues of length one are used. Because an index block is freed as soon
+        // as the read for the last value block is scheduled, the pipeline should not dry out even
+        // when switching between the tables.
+        //
+        // Note that level_{a,b}_position fields track the logical, deterministic progression of
+        // compaction.
+        //
+        // For output blocks:
+        // - the order of completions doesn't matter,
+        // - the blocks are not used after the completion of the IO. That is, only the number of
+        //   outstanding operations needs to be tracked. Use a RingBuffer with void elements rather
+        //   than an u32 for API uniformity and comptime upper bound. We have
+        //   <https://github.com/ziglang/zig/issues/3806> at home.
+        //
+        // In addition to static max size, the queues are additionally limited at runtime by the
+        // number of available free blocks. The queues are not limited by IOPS --- it is assumed
+        // that there are enough IOPS to fill up all the queues.
+        level_a_index_block: RingBufferType(*ResourcePool.Block, .{
+            .array = 1,
+        }) = .{ .buffer = undefined },
+
+        level_a_value_block: RingBufferType(*ResourcePool.Block, .{
+            .array = @divExact(constants.lsm_compaction_queue_read_max, 2),
+        }) = .{ .buffer = undefined },
+
+        level_b_index_block: RingBufferType(*ResourcePool.Block, .{
+            .array = 2, // Prefetch next table's index block
+        }) = .{ .buffer = undefined },
+
+        level_b_value_block: RingBufferType(*ResourcePool.Block, .{
+            .array = @divExact(constants.lsm_compaction_queue_read_max, 2),
+        }) = .{ .buffer = undefined },
+
+        output_blocks: RingBufferType(void, .{
+            .array = constants.lsm_compaction_queue_write_max,
+        }) = .{ .buffer = undefined },
+
+        pub fn init(tree: *Tree, grid: *Grid, level_b: u8) Compaction {
+            assert(level_b < constants.lsm_levels);
+
+            return Compaction{
+                .grid = grid,
+                .tree = tree,
+                .level_b = level_b,
+            };
+        }
+
+        pub fn reset(compaction: *Compaction) void {
+            compaction.grid.trace.cancel(.compact_beat);
+            compaction.grid.trace.cancel(.compact_beat_merge);
+            compaction.* = .{
+                .grid = compaction.grid,
+                .tree = compaction.tree,
+                .level_b = compaction.level_b,
+            };
+        }
+
+        /// Assert consistency of the compaction counters between beats. This isn't as
+        /// straightforward as calling counters.consistent() as we do at the end of the bar, since
+        /// we may carry over multiple input value blocks, and an output value block over to the
+        /// next beat.
+        ///
+        ///
+        /// Compute values_in, values_dropped, values_out, values_in_flight, and assert:
+        ///       values_out + values_in_flight == values_in - values_dropped
+        ///
+        // values_in: Values from the input value blocks that have been read from disk.
+        // values_dropped: Values dropped during merge.
+        // values_out: Values from the output value blocks that have been written to disk,
+        //             plus the values from the output value block being carried over to the
+        //             next beat (output of merge but not written to disk yet).
+        // values_in_flight: Values from the input value blocks being carried over to the next
+        //                   beat, minus the values that have already been consumed during
+        //                   merge.
+        fn assert_counter_consistency_between_beats(compaction: *const Compaction) void {
+            const values_in = compaction.counters.in;
+            const values_out = compaction.counters.out + compaction.table_builder.value_count;
+            const values_dropped = compaction.counters.dropped;
+
+            var values_in_flight: u64 = 0;
+
+            if (compaction.table_info_a.? == .immutable) {
+                assert(compaction.level_a_value_block.empty());
+                if (compaction.level_a_immutable_stage != .exhausted) {
+                    values_in_flight += compaction.table_info_a.?.immutable.len;
+                }
+            }
+
+            var level_a_value_block_iterator = compaction.level_a_value_block.iterator();
+            while (level_a_value_block_iterator.next()) |block| {
+                values_in_flight += Table.value_block_values_used(block.ptr).len;
+            }
+            values_in_flight -= compaction.level_a_position.value;
+
+            var level_b_value_block_iterator = compaction.level_b_value_block.iterator();
+            while (level_b_value_block_iterator.next()) |block| {
+                values_in_flight += Table.value_block_values_used(block.ptr).len;
+            }
+
+            values_in_flight -= compaction.level_b_position.value;
+
+            assert(values_out + values_in_flight == values_in - values_dropped);
+        }
+
+        pub fn assert_between_bars(compaction: *const Compaction) void {
+            assert(compaction.stage == .inactive);
+            assert(compaction.idle());
+            assert(compaction.block_queues_empty_output());
+            assert(compaction.block_queues_empty_input());
+
+            assert(compaction.table_builder.state == .no_blocks);
+            assert(compaction.table_builder_value_block == null);
+            assert(compaction.table_builder_index_block == null);
+            assert(compaction.manifest_entries.empty());
+        }
+
+        fn idle(compaction: *const Compaction) bool {
+            return compaction.pool == null and
+                compaction.callback == null and
+                compaction.quotas.beat_exhausted();
+        }
+
+        fn block_queues_empty_output(compaction: *const Compaction) bool {
+            return compaction.output_blocks.empty();
+        }
+
+        fn block_queues_empty_input(compaction: *const Compaction) bool {
+            return compaction.level_a_index_block.empty() and
+                compaction.level_a_value_block.empty() and
+                compaction.level_b_index_block.empty() and
+                compaction.level_b_value_block.empty();
+        }
+
+        /// Plan the work for the bar:
+        /// - check if compaction is needed at all (if the level_a is full),
+        /// - find a table on level_a and the corresponding range on level_b that should be
+        ///   compacted,
+        /// - compute the bar quota (just the total number of values in all input tables),
+        /// - execute move table optimization if range_b turns out to be empty.
+        pub fn bar_commence(compaction: *Compaction, op: u64) u64 {
+            assert(compaction.idle());
+            assert(compaction.block_queues_empty_output());
+            assert(compaction.block_queues_empty_input());
+
+            assert(compaction.stage == .inactive);
+            assert(op == compaction_op_min(op));
+
+            compaction.stage = .paused;
+            compaction.op_min = op;
+
+            if (compaction.level_b == 0) {
+                // Do not start compaction if the immutable table does not require compaction.
+                if (compaction.tree.table_immutable.mutability.immutable.flushed) {
+                    assert(compaction.quotas.bar == 0);
+                    assert(compaction.quotas.bar_exhausted());
+                    log.debug("{s}:{}: bar_commence: immutable table flushed", .{
+                        compaction.tree.config.name,
+                        compaction.level_b,
+                    });
+                    return 0;
+                }
+
+                const table_value_count_limit = Table.value_count_max;
+                assert(compaction.tree.table_immutable.count() > 0);
+                assert(compaction.tree.table_immutable.count() <= table_value_count_limit);
+
+                // If the mutable table will fit in the free capacity of the immutable table (even
+                // in the projected "worst" case of all full batches during the second half-bar),
+                // then defer compacting the immutable table into level 0.
+                //
+                // This optimization cannot apply to the last bar before a checkpoint trigger, since
+                // recovery from the checkpoint only replays that final bar, which must reconstruct
+                // the original immutable table.
+                // TODO(Snapshots) This optimization must be disabled to take a persistent snapshot.
+                const mutable_count_half_bar_first = compaction.tree.table_mutable.count();
+                const mutable_count_half_bar_last = @divExact(table_value_count_limit, 2);
+                const mutable_count = mutable_count_half_bar_first + mutable_count_half_bar_last;
+                const immutable_count = compaction.tree.table_immutable.count();
+                if (immutable_count + mutable_count <= table_value_count_limit) {
+                    const op_checkpoint =
+                        compaction.grid.superblock.working.vsr_state.checkpoint.header.op;
+                    const op_checkpoint_next = vsr.Checkpoint.checkpoint_after(op_checkpoint);
+                    const op_checkpoint_trigger_next =
+                        vsr.Checkpoint.trigger_for_checkpoint(op_checkpoint_next).?;
+                    const compaction_op_max = op + (half_bar_beat_count - 1);
+                    const last_half_bar_of_checkpoint =
+                        compaction_op_max == op_checkpoint_trigger_next;
+
+                    if (!last_half_bar_of_checkpoint) {
+                        assert(compaction.quotas.bar == 0);
+                        assert(compaction.quotas.bar_exhausted());
+                        log.debug("{s}:{}: bar_commence: immutable table flush skipped " ++
+                            "({}+{}+{} ≤ {})", .{
+                            compaction.tree.config.name,
+                            compaction.level_b,
+                            immutable_count,
+                            mutable_count_half_bar_first,
+                            mutable_count_half_bar_last,
+                            table_value_count_limit,
+                        });
+                        return 0;
+                    }
+                }
+
+                compaction.table_info_a = .{
+                    .immutable = compaction.tree.table_immutable.values_used(),
+                };
+
+                compaction.range_b = compaction.tree.manifest.immutable_table_compaction_range(
+                    compaction.tree.table_immutable.key_min(),
+                    compaction.tree.table_immutable.key_max(),
+                    .{ .value_count = compaction.tree.table_immutable.count() },
+                );
+
+                // +1 to count the immutable table (level A).
+                assert(compaction.range_b.?.tables.count() + 1 <= compaction_tables_input_max);
+                assert(compaction.range_b.?.key_min <= compaction.tree.table_immutable.key_min());
+                assert(compaction.tree.table_immutable.key_max() <= compaction.range_b.?.key_max);
+            } else {
+                const level_a = compaction.level_b - 1;
+
+                // Do not start compaction if level A does not require compaction.
+                const table_range = compaction.tree.manifest.compaction_table(level_a) orelse {
+                    assert(compaction.quotas.bar == 0);
+                    assert(compaction.quotas.bar_exhausted());
+                    log.debug("{s}:{}: bar_commence: nothing to compact", .{
+                        compaction.tree.config.name,
+                        compaction.level_b,
+                    });
+                    return 0;
+                };
+
+                compaction.table_info_a = .{ .disk = table_range.table_a };
+                compaction.range_b = table_range.range_b;
+
+                assert(compaction.range_b.?.tables.count() + 1 <= compaction_tables_input_max);
+                assert(compaction.table_info_a.?.disk.table_info.key_min <=
+                    compaction.table_info_a.?.disk.table_info.key_max);
+                assert(compaction.range_b.?.key_min <=
+                    compaction.table_info_a.?.disk.table_info.key_min);
+                assert(compaction.table_info_a.?.disk.table_info.key_max <=
+                    compaction.range_b.?.key_max);
+            }
+
+            switch (compaction.table_info_a.?) {
+                .immutable => {},
+                .disk => |table| {
+                    assert(!compaction.grid.free_set.is_released(table.table_info.address));
+                    assert(!compaction.grid.free_set.is_free(table.table_info.address));
+                },
+            }
+            for (compaction.range_b.?.tables.slice()) |table| {
+                assert(!compaction.grid.free_set.is_released(table.table_info.address));
+                assert(!compaction.grid.free_set.is_free(table.table_info.address));
+            }
+
+            var quota_bar = switch (compaction.table_info_a.?) {
+                .immutable => compaction.tree.table_immutable.count(),
+                .disk => |table| table.table_info.value_count,
+            };
+            for (compaction.range_b.?.tables.const_slice()) |*table| {
+                quota_bar += table.table_info.value_count;
+            }
+            compaction.quotas = .{
+                .beat = 0,
+                .beat_done = 0,
+                .bar = quota_bar,
+                .bar_done = 0,
+            };
+
+            log.debug("{s}:{}: bar_commence: quota_bar_done={} quota_bar={}", .{
+                compaction.tree.config.name,
+                compaction.level_b,
+                compaction.quotas.bar_done,
+                compaction.quotas.bar,
+            });
+            compaction.move_table = compaction.table_info_a.? == .disk and
+                compaction.range_b.?.tables.empty();
+            compaction.drop_tombstones = compaction.tree.manifest
+                .compaction_must_drop_tombstones(compaction.level_b, &compaction.range_b.?);
+
+            // The last level must always drop tombstones.
+            if (compaction.level_b == constants.lsm_levels - 1) assert(compaction.drop_tombstones);
+
+            assert(std.meta.eql(compaction.counters, .{}));
+            inline for (.{ compaction.level_a_position, compaction.level_b_position }) |position| {
+                assert(std.meta.eql(position, .{}));
+            }
+
+            // Append the entries to the manifest update queue here and now if we're doing
+            // move table. They'll be applied later by bar_complete().
+            if (compaction.move_table) {
+                const snapshot_max = snapshot_max_for_table_input(compaction.op_min);
+                assert(compaction.table_info_a.?.disk.table_info.snapshot_max >= snapshot_max);
+
+                compaction.manifest_entries.push(.{
+                    .operation = .move_to_level_b,
+                    .table = compaction.table_info_a.?.disk.table_info.*,
+                });
+
+                const value_count = compaction.table_info_a.?.disk.table_info.value_count;
+
+                compaction.quotas.beat = value_count;
+                compaction.quotas.beat_done = value_count;
+                compaction.quotas.bar_done = value_count;
+
+                assert(compaction.quotas.beat_exhausted());
+                assert(compaction.quotas.bar_exhausted());
+
+                return 0;
+            } else {
+                if (compaction.table_info_a.? == .immutable) {
+                    compaction.counters.in += compaction.table_info_a.?.immutable.len;
+                }
+                return compaction.quotas.bar;
+            }
+        }
+
+        /// Apply the changes that have been accumulated in memory to the manifest and remove any
+        /// tables that are now invisible.
+        pub fn bar_complete(compaction: *Compaction) void {
+            assert(compaction.idle());
+            assert(compaction.block_queues_empty_output());
+            assert(compaction.block_queues_empty_input());
+
+            assert(compaction.stage == .paused);
+            assert(compaction.counters.consistent());
+            assert(compaction.quotas.bar_exhausted());
+            // Assert blocks have been released back to the pipeline.
+            assert(compaction.table_builder.state == .no_blocks);
+            assert(compaction.table_builder_index_block == null);
+            assert(compaction.table_builder_value_block == null);
+
+            defer {
+                compaction.* = .{
+                    .grid = compaction.grid,
+                    .tree = compaction.tree,
+                    .level_b = compaction.level_b,
+                };
+                assert(compaction.stage == .inactive);
+            }
+
+            if (compaction.table_info_a == null) {
+                assert(compaction.range_b == null);
+                assert(compaction.manifest_entries.count() == 0);
+                assert(compaction.quotas.bar == 0);
+                if (compaction.level_b == 0) {
+                    // Either:
+                    // - the immutable table is empty (already flushed), or
+                    // - the mutable table will be absorbed into the immutable table.
+                    maybe(compaction.tree.table_immutable.mutability.immutable.flushed);
+                }
+                return;
+            }
+            assert(compaction.table_info_a != null);
+            assert(compaction.range_b != null);
+            assert(compaction.quotas.bar > 0);
+
+            switch (compaction.table_info_a.?) {
+                .immutable => {},
+                .disk => |table| {
+                    if (compaction.move_table) {
+                        assert(!compaction.grid.free_set.is_released(table.table_info.address));
+                        assert(!compaction.grid.free_set.is_free(table.table_info.address));
+                    } else {
+                        assert(compaction.grid.free_set.is_released(table.table_info.address));
+                    }
+                },
+            }
+            for (compaction.range_b.?.tables.slice()) |table| {
+                assert(compaction.grid.free_set.is_released(table.table_info.address));
+            }
+
+            log.debug("{s}:{}: bar_complete: " ++
+                "values_in={} values_out={} values_dropped={}", .{
+                compaction.tree.config.name,
+                compaction.level_b,
+                compaction.counters.in,
+                compaction.counters.out,
+                compaction.counters.dropped,
+            });
+
+            compaction.grid.trace.count(
+                .{ .compaction_values_physical = .{
+                    .tree = @enumFromInt(compaction.tree.config.id),
+                } },
+                compaction.counters.out,
+            );
+            if (compaction.level_b == 0) {
+                if (compaction.table_info_a.? == .immutable) {
+                    compaction.grid.trace.count(
+                        .{ .compaction_values_logical = .{
+                            .tree = @enumFromInt(compaction.tree.config.id),
+                        } },
+                        compaction.table_info_a.?.immutable.len,
+                    );
+                }
+            }
+
+            // Mark the immutable table as flushed, if we were compacting into level 0.
+            if (compaction.level_b == 0) {
+                assert(!compaction.tree.table_immutable.mutability.immutable.flushed);
+                compaction.tree.table_immutable.mutability.immutable.flushed = true;
+            }
+
+            // Each compaction's manifest updates are deferred to the end of the last
+            // bar to ensure:
+            // - manifest log updates are ordered deterministically relative to one another, and
+            // - manifest updates are not visible until after the blocks are all on disk.
+            const manifest = &compaction.tree.manifest;
+            const level_b = compaction.level_b;
+            const snapshot_max = snapshot_max_for_table_input(compaction.op_min);
+
+            var manifest_removed_value_count: u64 = 0;
+            var manifest_added_value_count: u64 = 0;
+
+            if (compaction.move_table) {
+                // If no compaction is required, don't update snapshot_max.
+            } else {
+                // These updates MUST precede insert_table() and move_table() since they use
+                // references to modify the ManifestLevel in-place.
+                switch (compaction.table_info_a.?) {
+                    .immutable => {
+                        manifest_removed_value_count = compaction.tree.table_immutable.count();
+                    },
+                    .disk => |table_info| {
+                        manifest_removed_value_count += table_info.table_info.value_count;
+                        manifest.update_table(level_b - 1, snapshot_max, table_info);
+                    },
+                }
+                for (compaction.range_b.?.tables.const_slice()) |table| {
+                    manifest_removed_value_count += table.table_info.value_count;
+                    manifest.update_table(level_b, snapshot_max, table);
+                }
+            }
+
+            for (compaction.manifest_entries.slice()) |*entry| {
+                switch (entry.operation) {
+                    .insert_to_level_b => {
+                        manifest.insert_table(level_b, &entry.table);
+                        manifest_added_value_count += entry.table.value_count;
+                    },
+                    .move_to_level_b => {
+                        manifest.move_table(level_b - 1, level_b, &entry.table);
+                        manifest_removed_value_count += entry.table.value_count;
+                        manifest_added_value_count += entry.table.value_count;
+                    },
+                }
+            }
+            if (compaction.move_table) {
+                assert(std.meta.eql(compaction.counters, .{}));
+                assert(manifest_added_value_count == manifest_removed_value_count);
+                assert(manifest_added_value_count > 0);
+            } else {
+                assert(manifest_added_value_count == compaction.counters.out);
+                assert(manifest_removed_value_count == compaction.counters.in);
+                assert(manifest_removed_value_count - manifest_added_value_count ==
+                    compaction.counters.dropped);
+            }
+
+            // Hide any tables that are now invisible.
+            manifest.remove_invisible_tables(
+                level_b,
+                &.{},
+                compaction.range_b.?.key_min,
+                compaction.range_b.?.key_max,
+            );
+            if (level_b > 0) {
+                manifest.remove_invisible_tables(
+                    level_b - 1,
+                    &.{},
+                    compaction.range_b.?.key_min,
+                    compaction.range_b.?.key_max,
+                );
+            }
+        }
+
+        pub fn beat_commence(
+            compaction: *Compaction,
+            values_count: u64,
+        ) void {
+            assert(compaction.idle());
+            assert(compaction.stage == .paused);
+            assert(compaction.block_queues_empty_output());
+            // We may be carrying over some blocks from the previous beat.
+            maybe(compaction.block_queues_empty_input());
+
+            if (compaction.move_table) assert(compaction.quotas.bar_exhausted());
+
+            // Run the compaction up to completion of the bar quota, if possible.
+            const values_remaining = (compaction.quotas.bar - compaction.quotas.bar_done);
+
+            compaction.quotas.beat = @min(values_count, values_remaining);
+            compaction.quotas.beat_done = 0;
+            assert(compaction.quotas.beat <= compaction.quotas.bar);
+        }
+
+        /// The entry point to the actual compaction work for the beat. Called by the forest.
+        pub fn compaction_dispatch_enter(
+            compaction: *Compaction,
+            options: struct {
+                pool: *ResourcePool,
+                callback: *const fn (pool: *ResourcePool, tree_id: u16, values_consumed: u64) void,
+            },
+        ) enum { pending, ready } {
+            assert(compaction.stage == .paused);
+            assert(compaction.block_queues_empty_output());
+            // We may be carrying over some blocks from the previous beat.
+            maybe(compaction.block_queues_empty_input());
+
+            if (compaction.move_table) assert(compaction.quotas.bar_exhausted());
+
+            if (compaction.quotas.bar_exhausted()) {
+                log.debug("{}: {s}:{}: beat_commence: bar quota={} fulfilled, done={}", .{
+                    compaction.grid.superblock.replica_index.?,
+                    compaction.tree.config.name,
+                    compaction.level_b,
+                    compaction.quotas.bar,
+                    compaction.quotas.bar_done,
+                });
+                return .ready;
+            }
+
+            if (compaction.quotas.beat_exhausted()) {
+                log.debug("{s}:{}: beat_commence: beat quota={} fulfilled, done={}", .{
+                    compaction.tree.config.name,
+                    compaction.level_b,
+                    compaction.quotas.beat,
+                    compaction.quotas.beat_done,
+                });
+                return .ready;
+            }
+
+            assert(!compaction.move_table);
+
+            compaction.grid.trace.start(.{ .compact_beat = .{
+                .tree = @enumFromInt(compaction.tree.config.id),
+                .level_b = compaction.level_b,
+            } });
+
+            assert(options.pool.idle());
+            assert(options.pool.grid_reservation != null);
+
+            compaction.pool = options.pool;
+            compaction.callback = options.callback;
+            compaction.stage = .beat;
+
+            compaction.compaction_dispatch();
+            return .pending;
+        }
+
+        // While beat_commence is called by the forest sequentially for each compaction, to get
+        // deterministic grid reservations, each compaction completes its own beat's work
+        // asynchronously
+        fn beat_complete(compaction: *Compaction) void {
+            assert(compaction.stage == .beat_quota_done);
+            switch (compaction.table_builder.state) {
+                .no_blocks => {},
+                .index_and_value_block => {
+                    assert(!compaction.table_builder.index_block_full());
+                    assert(!compaction.table_builder.value_block_full());
+                    assert(!compaction.quotas.bar_exhausted());
+                },
+                .index_block => {
+                    assert(!compaction.table_builder.index_block_full());
+                    assert(!compaction.quotas.bar_exhausted());
+                },
+            }
+
+            if (compaction.table_info_a.? == .immutable) {
+                switch (compaction.level_a_immutable_stage) {
+                    .ready, .exhausted => {},
+                    .merge => unreachable,
+                }
+            }
+
+            assert(compaction.block_queues_empty_output());
+            // We may be carrying over some input blocks to the next beat.
+            maybe(compaction.block_queues_empty_input());
+
+            compaction.assert_counter_consistency_between_beats();
+
+            assert(compaction.pool.?.idle());
+            maybe(compaction.pool.?.blocks_acquired() > 0);
+
+            if (compaction.quotas.bar_exhausted()) {
+                assert(compaction.table_builder.state == .no_blocks);
+                assert(compaction.table_builder_value_block == null);
+                assert(compaction.table_builder_index_block == null);
+                assert(compaction.block_queues_empty_input());
+            }
+
+            const pool = compaction.pool.?;
+            const callback = compaction.callback.?;
+
+            compaction.stage = .paused;
+            compaction.callback = null;
+            compaction.pool = null;
+
+            assert(compaction.idle());
+            assert(pool.idle());
+            log.debug("{s}:{}: beat_complete: quota_beat_done={} quota_beat={} " ++
+                "quota_bar_done={} quota_bar={}", .{
+                compaction.tree.config.name,
+                compaction.level_b,
+                compaction.quotas.beat_done,
+                compaction.quotas.beat,
+                compaction.quotas.bar_done,
+                compaction.quotas.bar,
+            });
+
+            callback(pool, compaction.tree.config.id, compaction.quotas.beat_done);
+        }
+
+        // Compaction is a lot of work: read input tables from both levels, merge their value
+        // blocks, write the results to disk. Many of these jobs can proceed in parallel. For
+        // example, only a single value block from each level is needed to start a merge.
+        //
+        // The job of compaction_dispatch is to kick off all the jobs. There are several additional
+        // concerns:
+        // - All jobs use the same common pool of resources (ResourcePool). The jobs are started
+        //   in the order that splits resources fairly (e.g., reads from level a and level b
+        //   alternate). Fairness also ensures that the process does not deadlock.
+        // - Jobs have dependencies --- merging needs value blocks, reading a value block needs the
+        //   corresponding index blocks.
+        // - A single bar of compaction should process only a fraction of the input, so the
+        //   processes can be suspended in the middle.
+        //
+        // A beat of compaction ends when both:
+        //   - at least quota.bar of input values is consumed,
+        //   - there's no incomplete output value blocks.
+        //
+        // In other words, the only compaction state that gets carried over to the next beat is a
+        // partially full index block. The current beat must end with writing a value block, and the
+        // next beat must start with re-reading level_a and level_b index and value blocks.
+        fn compaction_dispatch(compaction: *Compaction) void {
+            switch (compaction.stage) {
+                .beat,
+                .beat_quota_done,
+                => {},
+                .inactive,
+                .paused,
+                => unreachable,
+            }
+
+            // The loop below runs while (progressed) and, every time progressed is set to true,
+            // one of the safety_counter resources is acquired.
+            var progressed = true;
+            const safety_counter =
+                compaction.pool.?.reads.available() +
+                compaction.pool.?.writes.available() +
+                compaction.pool.?.cpus.available() + 1;
+            for (0..safety_counter) |_| {
+                if (!progressed) break;
+                progressed = false;
+
+                if (compaction.stage == .beat_quota_done) {
+                    // Just wait for all in-flight jobs to complete.
+                    return compaction.compaction_dispatch_beat_quota_done();
+                }
+
+                // To avoid deadlocks, allocate blocks for the table builder first.
+                if (compaction.table_builder.state == .no_blocks) {
+                    assert(compaction.table_builder_index_block == null);
+                    if (compaction.pool.?.block_acquire()) |block| {
+                        assert(block.stage == .free);
+                        block.stage = .build_index_block;
+                        compaction.table_builder.set_index_block(block.ptr);
+                        compaction.table_builder_index_block = block;
+                    } else {
+                        assert(compaction.output_blocks.count > 0);
+                    }
+                }
+
+                if (compaction.table_builder.state == .index_block) {
+                    assert(compaction.table_builder_value_block == null);
+                    if (compaction.pool.?.block_acquire()) |block| {
+                        assert(block.stage == .free);
+                        block.stage = .build_value_block;
+                        compaction.table_builder.set_value_block(block.ptr);
+                        compaction.table_builder_value_block = block;
+                    } else {
+                        assert(compaction.output_blocks.count > 0);
+                    }
+                }
+
+                const level_a_index_block_next =
+                    compaction.level_a_position.index_block +
+                    @as(u32, @intCast(compaction.level_a_index_block.count));
+                const level_b_index_block_next =
+                    compaction.level_b_position.index_block +
+                    @as(u32, @intCast(compaction.level_b_index_block.count));
+                const level_a_value_block_next =
+                    compaction.level_a_position.value_block +
+                    @as(u32, @intCast(compaction.level_a_value_block.count));
+                const level_b_value_block_next =
+                    compaction.level_b_position.value_block +
+                    @as(u32, @intCast(compaction.level_b_value_block.count));
+
+                // Read level A index block (for level_b > 0).
+                if (compaction.table_info_a.? == .disk) {
+                    assert(compaction.level_b > 0);
+                    if (!compaction.level_a_index_block.full() and
+                        level_a_index_block_next < 1)
+                    {
+                        if (compaction.pool.?.block_acquire()) |block| {
+                            const read = compaction.pool.?.reads.acquire().?;
+
+                            assert(block.stage == .free);
+                            block.stage = .read_index_block;
+                            compaction.level_a_index_block.push_assume_capacity(block);
+
+                            compaction.read_index_block(.level_a, read, block);
+                            progressed = true;
+                        } else {
+                            assert(compaction.level_a_index_block.count > 0 or
+                                compaction.output_blocks.count > 0);
+                        }
+                    }
+                }
+
+                // Read level B index block.
+                if (!compaction.level_b_index_block.full() and
+                    level_b_index_block_next < compaction.range_b.?.tables.count())
+                {
+                    if (compaction.pool.?.block_acquire()) |block| {
+                        const read = compaction.pool.?.reads.acquire().?;
+
+                        assert(block.stage == .free);
+                        block.stage = .read_index_block;
+                        compaction.level_b_index_block.push_assume_capacity(block);
+
+                        compaction.read_index_block(.level_b, read, block);
+                        progressed = true;
+                    } else {
+                        assert(compaction.level_b_index_block.count > 0 or
+                            compaction.output_blocks.count > 0);
+                    }
+                }
+
+                // Read level A value block.
+                if (compaction.table_info_a.? == .immutable) {
+                    // The whole table is in memory, no need to read anything.
+                    assert(compaction.level_a_index_block.count == 0);
+                } else {
+                    if (compaction.level_a_index_block.head()) |index_block| {
+                        if (index_block.stage == .read_index_block_done) {
+                            const index_schema = schema.TableIndex.from(index_block.ptr);
+                            const value_blocks_count =
+                                index_schema.value_blocks_used(index_block.ptr);
+                            if (!compaction.level_a_value_block.full() and
+                                level_a_value_block_next < value_blocks_count)
+                            {
+                                if (compaction.pool.?.block_acquire()) |block| {
+                                    const read = compaction.pool.?.reads.acquire().?;
+
+                                    assert(block.stage == .free);
+                                    block.stage = .read_value_block;
+                                    compaction.level_a_value_block.push_assume_capacity(block);
+
+                                    compaction.read_value_block(.level_a, read, block);
+                                    progressed = true;
+                                } else {
+                                    assert(compaction.level_a_value_block.count > 0 or
+                                        compaction.output_blocks.count > 0);
+                                }
+                            }
+                        } else {
+                            assert(index_block.stage == .read_index_block);
+                        }
+                    }
+                }
+
+                // Read level B value block.
+                if (compaction.level_b_index_block.head()) |index_block| {
+                    if (index_block.stage == .read_index_block_done) {
+                        const index_schema = schema.TableIndex.from(index_block.ptr);
+                        const value_blocks_count =
+                            index_schema.value_blocks_used(index_block.ptr);
+
+                        if (!compaction.level_b_value_block.full() and
+                            level_b_value_block_next < value_blocks_count)
+                        {
+                            if (compaction.pool.?.block_acquire()) |block| {
+                                const read = compaction.pool.?.reads.acquire().?;
+
+                                assert(block.stage == .free);
+                                block.stage = .read_value_block;
+                                compaction.level_b_value_block.push_assume_capacity(block);
+
+                                compaction.read_value_block(.level_b, read, block);
+                                progressed = true;
+                            } else {
+                                assert(compaction.level_b_value_block.count > 0 or
+                                    compaction.output_blocks.count > 0);
+                            }
+                        }
+                    } else {
+                        assert(index_block.stage == .read_index_block);
+                    }
+                }
+
+                const level_a_ready_immutable = compaction.table_info_a.? == .immutable and
+                    compaction.level_a_immutable_stage == .ready;
+                const level_a_ready_disk = compaction.table_info_a.? == .disk and
+                    compaction.level_a_value_block.head() != null and
+                    compaction.level_a_value_block.head().?.stage == .read_value_block_done;
+                const level_a_ready = level_a_ready_immutable or level_a_ready_disk;
+
+                const level_a_exhausted_immutable = compaction.table_info_a.? == .immutable and
+                    compaction.level_a_immutable_stage == .exhausted;
+                const level_a_exhausted_disk = compaction.table_info_a.? == .disk and
+                    compaction.level_a_index_block.count == 0 and
+                    compaction.level_a_value_block.count == 0;
+                const level_a_exhausted = level_a_exhausted_immutable or level_a_exhausted_disk;
+
+                const level_b_ready = compaction.level_b_value_block.head() != null and
+                    compaction.level_b_value_block.head().?.stage == .read_value_block_done;
+
+                const level_b_exhausted =
+                    compaction.level_b_index_block.count == 0 and
+                    compaction.level_b_value_block.count == 0;
+                const levels_exhausted = level_a_exhausted and level_b_exhausted;
+
+                assert(levels_exhausted == compaction.quotas.bar_exhausted());
+
+                if (compaction.table_builder.state == .index_and_value_block) {
+                    if (level_a_exhausted and level_b_exhausted) {
+                        assert(compaction.stage == .beat_quota_done);
+                    } else if ((level_a_exhausted or level_a_ready) and
+                        (level_b_exhausted or level_b_ready) and
+                        !compaction.table_builder.value_block_full())
+                    {
+                        const cpu = compaction.pool.?.cpus.acquire().?;
+                        compaction.merge(cpu);
+                        progressed = true;
+                    }
+
+                    // Write value and index blocks. It is important for correctness that both the
+                    // value block and index block are written together. Otherwise, we may end up
+                    // overflowing the index block's capacity for value block addresses.
+                    if (compaction.output_blocks.spare_capacity() >= 2) {
+                        if (compaction.table_builder.value_block_full()) {
+                            assert(!compaction.output_blocks.full());
+                            const write = compaction.pool.?.writes.acquire().?;
+                            compaction.write_value_block(write, .{
+                                .address = compaction.grid.acquire(
+                                    compaction.pool.?.grid_reservation.?,
+                                ),
+                            });
+                            progressed = true;
+                        }
+
+                        if (compaction.table_builder.index_block_full()) {
+                            assert(!compaction.output_blocks.full());
+                            const write = compaction.pool.?.writes.acquire().?;
+                            compaction.write_index_block(write, .{
+                                .address = compaction.grid.acquire(
+                                    compaction.pool.?.grid_reservation.?,
+                                ),
+                            });
+                            progressed = true;
+                        }
+                    }
+                }
+            } else unreachable;
+            assert(!progressed);
+            assert(!compaction.pool.?.idle());
+        }
+
+        fn compaction_dispatch_beat_quota_done(compaction: *Compaction) void {
+            assert(compaction.stage == .beat_quota_done);
+
+            if (compaction.table_builder.state == .index_and_value_block and
+                (compaction.table_builder.value_block_full() or compaction.quotas.bar_exhausted()))
+            {
+                if (compaction.table_builder.value_block_empty()) {
+                    assert(compaction.quotas.bar_exhausted());
+                    const value_block = compaction.table_builder_value_block.?;
+                    compaction.table_builder_value_block = null;
+                    compaction.table_builder.state = .index_block;
+                    assert(value_block.stage == .build_value_block);
+                    value_block.stage = .free;
+                    compaction.pool.?.block_release(value_block);
+                } else {
+                    if (!compaction.output_blocks.full()) {
+                        const write = compaction.pool.?.writes.acquire().?;
+                        compaction.write_value_block(write, .{
+                            .address = compaction.grid.acquire(
+                                compaction.pool.?.grid_reservation.?,
+                            ),
+                        });
+                        assert(compaction.table_builder.state == .index_block);
+                        assert(compaction.table_builder_value_block == null);
+                    }
+                }
+            }
+
+            if (compaction.table_builder.state == .index_block and
+                (compaction.table_builder.index_block_full() or compaction.quotas.bar_exhausted()))
+            {
+                if (compaction.table_builder.index_block_empty()) {
+                    assert(compaction.quotas.bar_exhausted());
+                    const index_block = compaction.table_builder_index_block.?;
+                    compaction.table_builder_index_block = null;
+                    compaction.table_builder.state = .no_blocks;
+                    assert(index_block.stage == .build_index_block);
+                    index_block.stage = .free;
+                    compaction.pool.?.block_release(index_block);
+                } else {
+                    if (!compaction.output_blocks.full()) {
+                        const write = compaction.pool.?.writes.acquire().?;
+                        compaction.write_index_block(write, .{
+                            .address = compaction.grid.acquire(
+                                compaction.pool.?.grid_reservation.?,
+                            ),
+                        });
+                        assert(compaction.table_builder.state == .no_blocks);
+                        assert(compaction.table_builder_value_block == null);
+                        assert(compaction.table_builder_index_block == null);
+                    }
+                }
+            }
+
+            if (compaction.output_blocks.count > 0) {
+                return;
+            }
+
+            switch (compaction.table_builder.state) {
+                .no_blocks => {},
+                .index_and_value_block => {
+                    assert(!compaction.table_builder.index_block_full());
+                    assert(!compaction.table_builder.value_block_full());
+                    assert(!compaction.quotas.bar_exhausted());
+                },
+                .index_block => {
+                    assert(!compaction.table_builder.index_block_full());
+                    assert(!compaction.quotas.bar_exhausted());
+                },
+            }
+
+            var level_a_value_block_iterator = compaction.level_a_value_block.iterator();
+            while (level_a_value_block_iterator.next()) |block| {
+                if (block.stage == .read_value_block) return;
+
+                assert(block.stage == .read_value_block_done);
+            }
+
+            var level_a_index_block_iterator = compaction.level_a_index_block.iterator();
+            while (level_a_index_block_iterator.next()) |block| {
+                if (block.stage == .read_index_block) return;
+
+                assert(block.stage == .read_index_block_done);
+            }
+
+            var level_b_value_block_iterator = compaction.level_b_value_block.iterator();
+            while (level_b_value_block_iterator.next()) |block| {
+                if (block.stage == .read_value_block) return;
+
+                assert(block.stage == .read_value_block_done);
+            }
+
+            var level_b_index_block_iterator = compaction.level_b_index_block.iterator();
+            while (level_b_index_block_iterator.next()) |block| {
+                if (block.stage == .read_index_block) return;
+
+                assert(block.stage == .read_index_block_done);
+            }
+
+            compaction.grid.trace.stop(.{ .compact_beat = .{
+                .tree = @enumFromInt(compaction.tree.config.id),
+                .level_b = compaction.level_b,
+            } });
+            compaction.beat_complete();
+        }
+
+        fn read_index_block(
+            compaction: *Compaction,
+            level: enum { level_a, level_b },
+            read: *ResourcePool.BlockRead,
+            index_block: *ResourcePool.Block,
+        ) void {
+            const level_b_index_block_next =
+                compaction.level_b_position.index_block +
+                @as(u32, @intCast(compaction.level_b_index_block.count));
+
+            assert(compaction.stage == .beat or compaction.stage == .beat_quota_done);
+            assert(index_block.stage == .read_index_block);
+            switch (level) {
+                .level_a => assert(compaction.level_a_position.index_block == 0),
+                .level_b => {
+                    assert(level_b_index_block_next - 1 < compaction.range_b.?.tables.count());
+                    assert(level_b_index_block_next > 0);
+                },
+            }
+
+            const table_ref = switch (level) {
+                .level_a => compaction.table_info_a.?.disk,
+                .level_b => compaction.range_b.?.tables.get(level_b_index_block_next - 1),
+            };
+            read.block = index_block;
+            read.compaction = compaction;
+            compaction.grid.read_block(
+                .{ .from_local_or_global_storage = read_index_block_callback },
+                &read.grid_read,
+                table_ref.table_info.address,
+                table_ref.table_info.checksum,
+                .{ .cache_read = true, .cache_write = true },
+            );
+        }
+
+        fn read_index_block_callback(grid_read: *Grid.Read, index_block: BlockPtrConst) void {
+            const read: *ResourcePool.BlockRead = @fieldParentPtr("grid_read", grid_read);
+            const compaction: *Compaction = read.parent(Compaction);
+            const block = read.block;
+            compaction.pool.?.reads.release(read);
+
+            assert(block.stage == .read_index_block);
+            stdx.copy_disjoint(.exact, u8, block.ptr, index_block);
+            block.stage = .read_index_block_done;
+            compaction.compaction_dispatch();
+        }
+
+        fn read_value_block(
+            compaction: *Compaction,
+            level: enum { level_a, level_b },
+            read: *ResourcePool.BlockRead,
+            value_block: *ResourcePool.Block,
+        ) void {
+            assert(compaction.stage == .beat or compaction.stage == .beat_quota_done);
+            assert(value_block.stage == .read_value_block);
+            if (level == .level_a) assert(compaction.table_info_a.? == .disk);
+
+            const index_block = switch (level) {
+                .level_a => compaction.level_a_index_block.head().?,
+                .level_b => compaction.level_b_index_block.head().?,
+            };
+
+            const level_a_value_block_next =
+                compaction.level_a_position.value_block +
+                @as(u32, @intCast(compaction.level_a_value_block.count));
+            const level_b_value_block_next =
+                compaction.level_b_position.value_block +
+                @as(u32, @intCast(compaction.level_b_value_block.count));
+
+            const value_block_index = blk: {
+                switch (level) {
+                    .level_a => {
+                        assert(level_a_value_block_next > 0);
+                        break :blk level_a_value_block_next - 1;
+                    },
+                    .level_b => {
+                        assert(level_b_value_block_next > 0);
+                        break :blk level_b_value_block_next - 1;
+                    },
+                }
+            };
+
+            const index_schema = schema.TableIndex.from(index_block.ptr);
+
+            const value_block_address =
+                index_schema.value_addresses_used(index_block.ptr)[value_block_index];
+            const value_block_checksum =
+                index_schema.value_checksums_used(index_block.ptr)[value_block_index];
+
+            read.block = value_block;
+            read.compaction = compaction;
+            compaction.grid.read_block(
+                .{ .from_local_or_global_storage = read_value_block_callback },
+                &read.grid_read,
+                value_block_address,
+                value_block_checksum.value,
+                .{ .cache_read = true, .cache_write = true },
+            );
+        }
+
+        // TODO: Support for LSM snapshots would require us to only remove blocks
+        // that are invisible.
+        fn read_value_block_release_table(
+            compaction: *Compaction,
+            index_block: BlockPtrConst,
+        ) void {
+            const index_schema = schema.TableIndex.from(index_block);
+            const index_block_address = Table.block_address(index_block);
+            const value_block_addresses = index_schema.value_addresses_used(index_block);
+
+            // Tables are released when the index block is no longer needed. Given that the same
+            // index block can get re-read across the bar, the same table can be released twice.
+            if (compaction.grid.free_set.is_released(index_block_address)) {
+                for (value_block_addresses) |address| {
+                    assert(compaction.grid.free_set.is_released(address));
+                }
+            } else {
+                compaction.grid.release(value_block_addresses);
+                compaction.grid.release(&.{index_block_address});
+            }
+        }
+
+        fn read_value_block_callback(grid_read: *Grid.Read, value_block: BlockPtrConst) void {
+            const read: *ResourcePool.BlockRead = @fieldParentPtr("grid_read", grid_read);
+            const compaction: *Compaction = read.parent(Compaction);
+            const block = read.block;
+            compaction.pool.?.reads.release(read);
+
+            assert(block.stage == .read_value_block);
+            stdx.copy_disjoint(.exact, u8, block.ptr, value_block);
+            block.stage = .read_value_block_done;
+            compaction.counters.in += Table.value_block_values_used(block.ptr).len;
+            compaction.compaction_dispatch();
+        }
+
+        fn merge(compaction: *Compaction, cpu: *ResourcePool.CPU) void {
+            assert(!compaction.quotas.bar_exhausted());
+
+            if (compaction.table_info_a.? == .immutable) {
+                if (compaction.level_a_immutable_stage == .ready) {
+                    compaction.level_a_immutable_stage = .merge;
+                } else assert(compaction.level_a_immutable_stage == .exhausted);
+            } else {
+                if (compaction.level_a_value_block.head()) |block| {
+                    assert(block.stage == .read_value_block_done);
+                    block.stage = .merge;
+                } else assert(compaction.level_b_value_block.head() != null);
+            }
+
+            if (compaction.level_b_value_block.head()) |block| {
+                assert(block.stage == .read_value_block_done);
+                block.stage = .merge;
+            }
+
+            assert(compaction.table_builder.state == .index_and_value_block);
+
+            cpu.compaction = compaction;
+            compaction.grid.on_next_tick(merge_callback, &cpu.next_tick);
+        }
+
+        fn merge_callback(next_tick: *Grid.NextTick) void {
+            const cpu: *ResourcePool.CPU = @fieldParentPtr("next_tick", next_tick);
+            const compaction: *Compaction = cpu.parent(Compaction);
+            compaction.pool.?.cpus.release(cpu);
+            assert(compaction.table_builder.state == .index_and_value_block);
+
+            compaction.grid.trace.start(.{ .compact_beat_merge = .{
+                .tree = @enumFromInt(compaction.tree.config.id),
+                .level_b = compaction.level_b,
+            } });
+
+            const values_source_a, const values_source_b = compaction.merge_inputs();
+            assert(values_source_a != null or values_source_b != null);
+
+            const values_target = compaction.table_builder
+                .value_block_values()[compaction.table_builder.value_count..];
+
+            inline for ([_]?[]const Value{
+                values_source_a,
+                values_source_b,
+                values_target,
+            }) |values_maybe| {
+                if (values_maybe) |values| {
+                    assert(values.len > 0);
+                    assert(values.len <= Table.data.value_count_max);
+                }
+            }
+
+            // Do the actual merge from inputs to the output (table builder).
+            const merge_result: MergeResult = if (values_source_a == null) blk: {
+                const consumed = values_copy(values_target, values_source_b.?);
+                break :blk .{
+                    .consumed_a = 0,
+                    .consumed_b = consumed,
+                    .dropped = 0,
+                    .produced = consumed,
+                };
+            } else if (values_source_b == null) blk: {
+                if (compaction.drop_tombstones) {
+                    const copy_result = values_copy_drop_tombstones(
+                        values_target,
+                        values_source_a.?,
+                    );
+                    break :blk .{
+                        .consumed_a = copy_result.consumed,
+                        .consumed_b = 0,
+                        .dropped = copy_result.dropped,
+                        .produced = copy_result.produced,
+                    };
+                } else {
+                    const consumed = values_copy(values_target, values_source_a.?);
+                    break :blk .{
+                        .consumed_a = consumed,
+                        .consumed_b = 0,
+                        .dropped = 0,
+                        .produced = consumed,
+                    };
+                }
+            } else values_merge(
+                values_target,
+                values_source_a.?,
+                values_source_b.?,
+                compaction.drop_tombstones,
+            );
+
+            compaction.level_a_position.value += merge_result.consumed_a;
+            compaction.level_b_position.value += merge_result.consumed_b;
+            compaction.table_builder.value_count += merge_result.produced;
+
+            if (compaction.table_info_a.? == .immutable) {
+                assert(compaction.level_a_position.value <= Table.value_count_max);
+            } else {
+                assert(compaction.level_a_position.value <= Table.data.value_count_max);
+            }
+            assert(compaction.level_b_position.value <= Table.data.value_count_max);
+            assert(compaction.table_builder.value_count <= Table.data.value_count_max);
+
+            const consumed_ab = merge_result.consumed_a + merge_result.consumed_b;
+
+            compaction.quotas.bar_done += consumed_ab;
+            compaction.quotas.beat_done += consumed_ab;
+
+            compaction.counters.dropped += merge_result.dropped;
+
+            assert(compaction.quotas.bar_done <= compaction.quotas.bar);
+
+            compaction.merge_advance_position();
+
+            // NB: although all the work here is synchronous, we don't defer trace.stop precisely
+            // to exclude compaction.dispatch call below.
+            compaction.grid.trace.stop(.{ .compact_beat_merge = .{
+                .tree = @enumFromInt(compaction.tree.config.id),
+                .level_b = compaction.level_b,
+            } });
+            compaction.compaction_dispatch();
+        }
+
+        fn merge_inputs(compaction: *const Compaction) struct { ?[]const Value, ?[]const Value } {
+            const level_a_values_used: ?[]const Value = values: {
+                switch (compaction.table_info_a.?) {
+                    .immutable => {
+                        if (compaction.level_a_immutable_stage == .merge) {
+                            break :values compaction.table_info_a.?.immutable;
+                        } else {
+                            assert(compaction.level_a_immutable_stage == .exhausted);
+                            break :values null;
+                        }
+                    },
+                    .disk => {
+                        if (compaction.level_a_value_block.head()) |block| {
+                            assert(block.stage == .merge);
+                            break :values Table.value_block_values_used(block.ptr);
+                        } else {
+                            break :values null;
+                        }
+                    },
+                }
+            };
+
+            const level_b_values_used: ?[]const Value = values: {
+                if (compaction.level_b_value_block.head()) |block| {
+                    assert(block.stage == .merge);
+                    break :values Table.value_block_values_used(block.ptr);
+                } else {
+                    break :values null;
+                }
+            };
+            assert(!(level_a_values_used == null and level_b_values_used == null));
+
+            const level_a_values = if (level_a_values_used) |values_used| values: {
+                const values_remaining = values_used[compaction.level_a_position.value..];
+                // Only consume one block at a time so that a beat never outputs past its quota
+                // by more than one value block.
+                const limit = @min(
+                    Table.data.value_count_max,
+                    values_remaining.len,
+                );
+                break :values values_remaining[0..limit];
+            } else null;
+
+            const level_b_values = if (level_b_values_used) |values_used|
+                values_used[compaction.level_b_position.value..]
+            else
+                null;
+
+            return .{ level_a_values, level_b_values };
+        }
+
+        // merge_callback advances just position.values. Here, we implement the carry-flag logic,
+        // advancing value_block and index_block. This is also the place where determine that the
+        // beat's quota of work is done and begin to wind down the dispatch loop.
+        fn merge_advance_position(compaction: *Compaction) void {
+            if (compaction.table_info_a.? == .immutable) {
+                if (compaction.level_a_immutable_stage == .merge) {
+                    if (compaction.level_a_position.value ==
+                        compaction.table_info_a.?.immutable.len)
+                    {
+                        compaction.level_a_position.value_block += 1;
+                        assert(compaction.level_a_position.value_block == 1);
+                        compaction.level_a_position.value = 0;
+                        compaction.level_a_immutable_stage = .exhausted;
+                    } else {
+                        compaction.level_a_immutable_stage = .ready;
+                    }
+                } else {
+                    assert(compaction.level_a_immutable_stage == .exhausted);
+                }
+            } else {
+                if (compaction.level_a_value_block.head()) |value_block| {
+                    assert(value_block.stage == .merge);
+                    if (compaction.level_a_position.value ==
+                        Table.value_block_values_used(value_block.ptr).len)
+                    {
+                        _ = compaction.level_a_value_block.pop();
+
+                        compaction.level_a_position.value_block += 1;
+                        compaction.level_a_position.value = 0;
+
+                        const index_block = compaction.level_a_index_block.head().?;
+                        assert(index_block.stage == .read_index_block_done);
+                        const index_schema = schema.TableIndex.from(index_block.ptr);
+                        const value_blocks_count =
+                            index_schema.value_blocks_used(index_block.ptr);
+
+                        // It is imperative that we pop the index block when the final value block
+                        // is popped. While it is tempting to pop the index block when we issue
+                        // a read for the final value block, this would be incorrect as it would
+                        // lead to an incorrect index being computed for level_a_value_block_next
+                        // in `compaction_dispatch`.
+                        if (compaction.level_a_position.value_block == value_blocks_count) {
+                            compaction.level_a_position.index_block += 1;
+                            assert(compaction.level_a_position.index_block == 1);
+                            compaction.level_a_position.value_block = 0;
+
+                            const popped = compaction.level_a_index_block.pop().?;
+                            assert(popped == index_block);
+                            compaction.read_value_block_release_table(index_block.ptr);
+                            index_block.stage = .free;
+                            compaction.pool.?.block_release(index_block);
+                        }
+
+                        value_block.stage = .free;
+                        compaction.pool.?.block_release(value_block);
+                    } else {
+                        value_block.stage = .read_value_block_done;
+                    }
+                } else {
+                    assert(compaction.level_a_position.value == 0); // Level A exhausted.
+                }
+            }
+
+            if (compaction.level_b_value_block.head()) |value_block| {
+                assert(value_block.stage == .merge);
+                if (compaction.level_b_position.value ==
+                    Table.value_block_values_used(value_block.ptr).len)
+                {
+                    _ = compaction.level_b_value_block.pop().?;
+                    compaction.level_b_position.value_block += 1;
+                    compaction.level_b_position.value = 0;
+
+                    const index_block = compaction.level_b_index_block.head().?;
+                    assert(index_block.stage == .read_index_block_done);
+                    const index_schema = schema.TableIndex.from(index_block.ptr);
+                    const value_blocks_count =
+                        index_schema.value_blocks_used(index_block.ptr);
+
+                    // It is imperative that we pop the index block when the final value block
+                    // is popped. While it is tempting to pop the index block when we issue
+                    // a read for the final value block, this would be incorrect as it would
+                    // lead to an incorrect index being computed for level_b_value_block_next
+                    // in `compaction_dispatch`.
+                    if (compaction.level_b_position.value_block == value_blocks_count) {
+                        compaction.level_b_position.index_block += 1;
+                        compaction.level_b_position.value_block = 0;
+
+                        const popped = compaction.level_b_index_block.pop().?;
+                        assert(popped == index_block);
+                        compaction.read_value_block_release_table(index_block.ptr);
+                        index_block.stage = .free;
+
+                        compaction.pool.?.block_release(index_block);
+                    }
+
+                    value_block.stage = .free;
+                    compaction.pool.?.block_release(value_block);
+                } else {
+                    value_block.stage = .read_value_block_done;
+                }
+            } else {
+                assert(compaction.level_b_position.value == 0); // Level B exhausted.
+            }
+
+            if (compaction.quotas.beat_exhausted()) {
+                assert(compaction.stage == .beat);
+                compaction.stage = .beat_quota_done;
+            }
+        }
+
+        fn write_value_block(
+            compaction: *Compaction,
+            write: *ResourcePool.BlockWrite,
+            options: struct { address: u64 },
+        ) void {
+            const block = compaction.table_builder_value_block.?;
+            assert(block.stage == .build_value_block);
+            assert(compaction.table_builder.value_block == block.ptr);
+            assert(!compaction.output_blocks.full());
+
+            compaction.counters.out += compaction.table_builder.value_count;
+            compaction.table_builder.value_block_finish(.{
+                .cluster = compaction.grid.superblock.working.cluster,
+                .release = compaction.grid.superblock.working.vsr_state.checkpoint.release,
+                .address = options.address,
+                .snapshot_min = snapshot_min_for_table_output(compaction.op_min),
+                .tree_id = compaction.tree.config.id,
+            });
+            assert(compaction.table_builder.state == .index_block);
+            compaction.table_builder_value_block = null;
+
+            compaction.output_blocks.push_assume_capacity({});
+            block.stage = .write_value_block;
+
+            write.block = block;
+            write.compaction = compaction;
+            compaction.grid.create_block(write_block_callback, &write.grid_write, &write.block.ptr);
+        }
+
+        fn write_index_block(
+            compaction: *Compaction,
+            write: *ResourcePool.BlockWrite,
+            options: struct { address: u64 },
+        ) void {
+            const block = compaction.table_builder_index_block.?;
+            assert(block.stage == .build_index_block);
+            assert(compaction.table_builder.index_block == block.ptr);
+            assert(!compaction.output_blocks.full());
+
+            const table = compaction.table_builder.index_block_finish(.{
+                .cluster = compaction.grid.superblock.working.cluster,
+                .release = compaction.grid.superblock.working.vsr_state.checkpoint.release,
+                .address = options.address,
+                .snapshot_min = snapshot_min_for_table_output(compaction.op_min),
+                .tree_id = compaction.tree.config.id,
+            });
+            assert(compaction.table_builder.state == .no_blocks);
+            compaction.table_builder_index_block = null;
+
+            compaction.manifest_entries.push(.{
+                .operation = .insert_to_level_b,
+                .table = table,
+            });
+
+            compaction.output_blocks.push_assume_capacity({});
+            block.stage = .write_index_block;
+
+            write.block = block;
+            write.compaction = compaction;
+            compaction.grid.create_block(write_block_callback, &write.grid_write, &write.block.ptr);
+        }
+
+        fn write_block_callback(grid_write: *Grid.Write) void {
+            const write: *ResourcePool.BlockWrite = @fieldParentPtr("grid_write", grid_write);
+            const compaction: *Compaction = write.parent(Compaction);
+            const block = write.block;
+            compaction.pool.?.writes.release(write);
+
+            assert(block.stage == .write_value_block or block.stage == .write_index_block);
+            block.stage = .free;
+            compaction.pool.?.block_release(block);
+
+            const popped = compaction.output_blocks.pop();
+            assert(popped != null);
+
+            compaction.compaction_dispatch();
+        }
+
+        // The three functions below are hot CPU loops doing the actual merging, TigerBeetle's data
+        // plane. To reduce the probability of the optimizer getting confused over pointers, don't
+        // use 'self' and instead specify all inputs and outputs explicitly. Its the caller's job to
+        // apply control plane changes to the compaction state.
+        //
+        // TODO: Add micro benchmarks.
+
+        fn values_copy(values_target: []Value, values_source: []const Value) u32 {
+            assert(values_source.len > 0);
+            assert(values_source.len <= Table.data.value_count_max);
+            assert(values_target.len > 0);
+            assert(values_target.len <= Table.data.value_count_max);
+
+            const len: u32 = @intCast(@min(values_source.len, values_target.len));
+            stdx.copy_disjoint(
+                .exact,
+                Value,
+                values_target[0..len],
+                values_source[0..len],
+            );
+
+            return len;
+        }
+
+        const CopyDropTombstonesResult = struct {
+            consumed: u32,
+            dropped: u32,
+            produced: u32,
+        };
+        /// Copy values from values_source to values_target, dropping tombstones as we go.
+        fn values_copy_drop_tombstones(
+            values_target: []Value,
+            values_source: []const Value,
+        ) CopyDropTombstonesResult {
+            assert(values_source.len > 0);
+            assert(values_source.len <= Table.data.value_count_max);
+            assert(values_target.len > 0);
+            assert(values_target.len <= Table.data.value_count_max);
+
+            var index_source: usize = 0;
+            var index_target: usize = 0;
+            // Merge as many values as possible.
+            while (index_source < values_source.len and
+                index_target < values_target.len)
+            {
+                const value_in = &values_source[index_source];
+                index_source += 1;
+                if (tombstone(value_in)) {
+                    assert(Table.usage != .secondary_index);
+                    continue;
+                }
+                values_target[index_target] = value_in.*;
+                index_target += 1;
+            }
+            const copy_result: CopyDropTombstonesResult = .{
+                .consumed = @intCast(index_source),
+                .dropped = @intCast(index_source - index_target),
+                .produced = @intCast(index_target),
+            };
+            assert(copy_result.consumed > 0);
+            assert(copy_result.consumed <= values_source.len);
+            assert(copy_result.dropped <= copy_result.consumed);
+            assert(copy_result.produced <= values_target.len);
+            assert(copy_result.produced == copy_result.consumed - copy_result.dropped);
+            return copy_result;
+        }
+
+        const MergeResult = struct {
+            consumed_a: u32,
+            consumed_b: u32,
+            dropped: u32,
+            produced: u32,
+        };
+
+        /// Merge values from table_a and table_b, with table_a taking precedence. Tombstones may
+        /// or may not be dropped depending on bar.drop_tombstones.
+        fn values_merge(
+            values_target: []Value,
+            values_source_a: []const Value,
+            values_source_b: []const Value,
+            drop_tombstones: bool,
+        ) MergeResult {
+            assert(values_source_a.len > 0);
+            assert(values_source_a.len <= Table.data.value_count_max);
+            assert(values_source_b.len > 0);
+            assert(values_source_b.len <= Table.data.value_count_max);
+            assert(values_target.len > 0);
+            assert(values_target.len <= Table.data.value_count_max);
+
+            var index_source_a: usize = 0;
+            var index_source_b: usize = 0;
+            var index_target: usize = 0;
+
+            while (index_source_a < values_source_a.len and
+                index_source_b < values_source_b.len and
+                index_target < values_target.len)
+            {
+                const value_a = &values_source_a[index_source_a];
+                const value_b = &values_source_b[index_source_b];
+                switch (std.math.order(key_from_value(value_a), key_from_value(value_b))) {
+                    .lt => { // Pick value from level a.
+                        index_source_a += 1;
+                        if (drop_tombstones and tombstone(value_a)) {
+                            assert(Table.usage != .secondary_index);
+                            continue;
+                        }
+                        values_target[index_target] = value_a.*;
+                        index_target += 1;
+                    },
+                    .gt => { // Pick value from level b.
+                        index_source_b += 1;
+                        values_target[index_target] = value_b.*;
+                        index_target += 1;
+                    },
+                    .eq => { // Values have equal keys -- collapse them!
+                        index_source_a += 1;
+                        index_source_b += 1;
+
+                        if (comptime Table.usage == .secondary_index) {
+                            // Secondary index optimization --- cancel out put and remove.
+                            assert(tombstone(value_a) != tombstone(value_b));
+                        } else {
+                            if (drop_tombstones and tombstone(value_a)) continue;
+                            values_target[index_target] = value_a.*;
+                            index_target += 1;
+                        }
+                    },
+                }
+            }
+
+            const merge_result: MergeResult = .{
+                .consumed_a = @intCast(index_source_a),
+                .consumed_b = @intCast(index_source_b),
+                .dropped = @intCast(index_source_a + index_source_b - index_target),
+                .produced = @intCast(index_target),
+            };
+            assert(merge_result.consumed_a > 0 or merge_result.consumed_b > 0);
+            assert(merge_result.consumed_a <= values_source_a.len);
+            assert(merge_result.consumed_b <= values_source_b.len);
+            assert(merge_result.dropped <= merge_result.consumed_a + merge_result.consumed_b);
+            assert(merge_result.produced <= values_target.len);
+            assert(merge_result.produced ==
+                merge_result.consumed_a + merge_result.consumed_b - merge_result.dropped);
+            return merge_result;
+        }
+    };
+}
+
+pub fn snapshot_max_for_table_input(op_min: u64) u64 {
+    return snapshot_min_for_table_output(op_min) - 1;
+}
+
+pub fn snapshot_min_for_table_output(op_min: u64) u64 {
+    assert(op_min > 0);
+    assert(op_min % @divExact(constants.lsm_compaction_ops, 2) == 0);
+    return op_min + @divExact(constants.lsm_compaction_ops, 2);
+}
+
+/// Returns the first op of the compaction (Compaction.op_min) for a given op/beat.
+///
+/// After this compaction finishes:
+/// - `op_min + half_bar_beat_count - 1` will be the input tables' snapshot_max.
+/// - `op_min + half_bar_beat_count` will be the output tables' snapshot_min.
+///
+/// Each half-bar has a separate op_min (for deriving the output snapshot_min) instead of each full
+/// bar because this allows the output tables of the first half-bar's compaction to be prefetched
+/// against earlier — hopefully while they are still warm in the cache from being written.
+///
+///
+/// These charts depict the commit/compact ops over a series of
+/// commits and compactions (with lsm_compaction_ops=8).
+///
+/// Legend:
+///
+///   ┼   full bar (first half-bar start)
+///   ┬   half bar (second half-bar start)
+///       This is incremented at the end of each compact().
+///   .   op is in mutable table (in memory)
+///   ,   op is in immutable table (in memory)
+///   #   op is on disk
+///   ✓   checkpoint() may follow compact()
+///
+///   0 2 4 6 8 0 2 4 6
+///   ┼───┬───┼───┬───┼
+///   .       ╷       ╷     init(superblock.commit_min=0)⎤ Compaction is effectively a noop for the
+///   ..      ╷       ╷     commit;compact( 1) start/end ⎥ first bar because there are no tables on
+///   ...     ╷       ╷     commit;compact( 2) start/end ⎥ disk yet, and no immutable table to
+///   ....    ╷       ╷     commit;compact( 3) start/end ⎥ flush.
+///   .....   ╷       ╷     commit;compact( 4) start/end ⎥
+///   ......  ╷       ╷     commit;compact( 5) start/end ⎥ This applies:
+///   ....... ╷       ╷     commit;compact( 6) start/end ⎥ - when the LSM is starting on a freshly
+///   ........╷       ╷     commit;compact( 7) start    ⎤⎥   formatted data file, and also
+///   ,,,,,,,,.       ╷  ✓         compact( 7)       end⎦⎦ - when the LSM is recovering from a crash
+///   ,,,,,,,,.       ╷     commit;compact( 8) start/end     (see below).
+///   ,,,,,,,,..      ╷     commit;compact( 9) start/end
+///   ,,,,,,,,...     ╷     commit;compact(10) start/end
+///   ,,,,,,,,....    ╷     commit;compact(11) start/end
+///   ,,,,,,,,.....   ╷     commit;compact(12) start/end
+///   ,,,,,,,,......  ╷     commit;compact(13) start/end
+///   ,,,,,,,,....... ╷     commit;compact(14) start/end
+///   ,,,,,,,,........╷     commit;compact(15) start    ⎤
+///   ########,,,,,,,,╷  ✓         compact(15)       end⎦
+///   ########,,,,,,,,.     commit;compact(16) start/end
+///   ┼───┬───┼───┬───┼
+///   0 2 4 6 8 0 2 4 6
+///   ┼───┬───┼───┬───┼                                    Recover with a checkpoint taken at op 15.
+///   ########        ╷     init(superblock.commit_min=7)  At op 15, ops 8…15 are in memory, so they
+///   ########.       ╷     commit        ( 8) start/end ⎤ were dropped by the crash.
+///   ########..      ╷     commit        ( 9) start/end ⎥
+///   ########...     ╷     commit        (10) start/end ⎥ But compaction is not run for ops 8…15
+///   ########....    ╷     commit        (11) start/end ⎥ because it was already performed
+///   ########.....   ╷     commit        (12) start/end ⎥ before the checkpoint.
+///   ########......  ╷     commit        (13) start/end ⎥
+///   ########....... ╷     commit        (14) start/end ⎥ We can begin to compact again at op 16,
+///   ########........╷     commit        (15) start    ⎤⎥ because those compactions (if previously
+///   ########,,,,,,,,╷  ✓                (15)       end⎦⎦ performed) are not included in the
+///   ########,,,,,,,,.     commit;compact(16) start/end   checkpoint.
+///   ┼───┬───┼───┬───┼
+///   0 2 4 6 8 0 2 4 6
+///
+/// Notice how in the checkpoint recovery example above, we are careful not to `compact(op)` twice
+/// for any op (even if we crash/recover), since that could lead to differences between replicas'
+/// storage. The last bar of `commit()`s is always only in memory, so it is safe to repeat.
+pub fn compaction_op_min(op: u64) u64 {
+    assert(op >= half_bar_beat_count);
+    return op - op % half_bar_beat_count;
+}