tigerbeetle 0.0.38 → 0.0.39

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 55a5b0b68111a9aeac3084772fb0f0ad185a67d9f41c67b5bcd6392fd7fcab61
-  data.tar.gz: '03976907f14a64b11b0fd9f7bf0693a4f44a887b997b34c9912bc71b1653eea5'
+  metadata.gz: 234689582eb24285e6dced340dceaad9abc90f259512ac12343b796f7daa1088
+  data.tar.gz: 0f402fb280af28464ca401a00ec7ad8d2e06b1cf5d8a0e7253c2a4ae147a20dd
 SHA512:
-  metadata.gz: 8c256706f6343e8e3d0cc1d130482009b28cdcc696ababa97a0d75f4fb7b74e7aa880fa364cb0ff55e274e26b876242f5f3418e6090f9e5dd5bdf17698252737
-  data.tar.gz: 7683f0b905a05ec28a2b164bc8a18f285ef64be2b6190ae40503b8f9ade3864238047ac5f53e6e0c5c60e94347a1483b1ad8a1b79221ae688941673bade1f82a
+  metadata.gz: da4ee2ecd0088a903b1b0f4341f2e8fc68d4be85bf990aa7b2caeeb99d04fecec75f2c3412124dcea3d345915ed4a55a534ce860a694f43f8e752fef0d97af96
+  data.tar.gz: 4a8e2049ff1db496e3b1eb33ed37b7ae7f6216f9963be9db27e90eac579251e1686e2e837df7430d190d4a5768e2ca4ae5f03627a47dbc2d65f1423be69a1f29

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 0.0.39
+
+- Based on TigerBeetle 0.16.75
+
+
 ## 0.0.38
 
 - Based on TigerBeetle 0.16.74

data/ext/tb_client/tigerbeetle/build.zig

@@ -2272,55 +2272,24 @@ fn fetch(b: *std.Build, options: struct {
     file_name: []const u8,
     hash: ?[]const u8,
 }) std.Build.LazyPath {
-    const copy_from_cache = b.addRunArtifact(b.addExecutable(.{
-        .name = "copy-from-cache",
+    const fetch_step = b.addRunArtifact(b.addExecutable(.{
+        .name = "fetch",
         .root_module = b.createModule(.{
-            .root_source_file = b.addWriteFiles().add("main.zig",
-                \\const builtin = @import("builtin");
-                \\const std = @import("std");
-                \\const assert = std.debug.assert;
-                \\
-                \\pub fn main() !void {
-                \\    var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
-                \\    const allocator = arena.allocator();
-                \\    const args = try std.process.argsAlloc(allocator);
-                \\    assert(args.len == 5 or args.len == 6);
-                \\
-                \\    const hash_and_newline = try std.fs.cwd().readFileAlloc(allocator, args[2], 128);
-                \\    assert(hash_and_newline[hash_and_newline.len - 1] == '\n');
-                \\    const hash = hash_and_newline[0 .. hash_and_newline.len - 1];
-                \\    if (args.len == 6 and !std.mem.eql(u8, args[5], hash)) {
-                \\        std.debug.panic(
-                \\            \\bad hash
-                \\            \\specified:  {s}
-                \\            \\downloaded: {s}
-                \\            \\
-                \\        , .{ args[5], hash });
-                \\    }
-                \\
-                \\    const source_path = try std.fs.path.join(allocator, &.{ args[1], hash, args[3] });
-                \\    try std.fs.cwd().copyFile(
-                \\        source_path,
-                \\        std.fs.cwd(),
-                \\        args[4],
-                \\        .{},
-                \\    );
-                \\}
-            ),
+            .root_source_file = b.path("./src/build/fetch.zig"),
             .target = b.graph.host,
         }),
     }));
-    copy_from_cache.addArg(
-        b.graph.global_cache_root.join(b.allocator, &.{"p"}) catch @panic("OOM"),
-    );
-    copy_from_cache.addFileArg(
-        b.addSystemCommand(&.{ b.graph.zig_exe, "fetch", options.url }).captureStdOut(),
-    );
-    copy_from_cache.addArg(options.file_name);
-    const result = copy_from_cache.addOutputFileArg(options.file_name);
-    if (options.hash) |hash| {
-        copy_from_cache.addArg(hash);
-    }
+    fetch_step.setName(b.fmt("fetch {s}", .{options.url}));
+
+    fetch_step.addArgs(&.{
+        b.graph.zig_exe,
+        b.graph.global_cache_root.path orelse ".",
+        options.url,
+        options.file_name,
+    });
+    const result = fetch_step.addOutputFileArg(options.file_name);
+    if (options.hash) |hash| fetch_step.addArg(hash);
+
     return result;
 }
 

data/ext/tb_client/tigerbeetle/src/build/fetch.zig

@@ -0,0 +1,112 @@
+const std = @import("std");
+const assert = std.debug.assert;
+const Allocator = std.mem.Allocator;
+
+const log = std.log;
+
+pub const std_options: std.Options = .{
+    .log_level = .info,
+};
+
+pub fn main() !void {
+    var arena_instance = std.heap.ArenaAllocator.init(std.heap.page_allocator);
+    const arena = arena_instance.allocator();
+
+    const args = try std.process.argsAlloc(arena);
+    assert(args.len == 6 or args.len == 7);
+
+    _, const zig, const global_cache, const url, const file_name, const out = args[0..6].*;
+    const hash_optional = if (args.len == 7) args[6] else null;
+    assert(args.len <= 7);
+
+    if (hash_optional) |hash| {
+        // Fast path --- don't touch the Internet if we have the hash locally.
+        const cached = path_join(arena, &.{ global_cache, "p", hash, file_name });
+        if (std.fs.cwd().copyFile(cached, std.fs.cwd(), out, .{})) {
+            log.debug("download skipped: cache hit", .{});
+            return;
+        } else |_| { // Time to ask for forgiveness!
+            log.debug("download: cache miss", .{});
+        }
+    } else {
+        log.debug("download: no hash", .{});
+    }
+
+    // If curl is available, use it for robust downloads, and then
+    // `zig fetch` a local file to get the hash. Otherwise, fetch
+    // the url directly.
+    const hash = if (exec_ok(arena, &.{ "curl", "--version" })) hash: {
+        log.debug("download: curl", .{});
+        const url_file_name = url[std.mem.lastIndexOf(u8, url, "/").?..];
+        const tmp_dir = path_join(arena, &.{
+            global_cache,
+            "tmp",
+            &std.fmt.bytesToHex(std.mem.asBytes(&std.crypto.random.int(u64)), .lower),
+        });
+        defer std.fs.cwd().deleteTree(tmp_dir) catch {};
+
+        try std.fs.cwd().makePath(tmp_dir);
+
+        const curl_output = path_join(arena, &.{ tmp_dir, url_file_name });
+        _ = try exec(arena, &(.{
+            "curl",             "--retry-all-errors",
+            "--retry",          "5",
+            "--retry-max-time", "120",
+            "--location",       url,
+            "--output",         curl_output,
+        }));
+        break :hash try exec(arena, &.{ zig, "fetch", curl_output });
+    } else hash: {
+        log.debug("download: zig fetch", .{});
+        break :hash try exec(arena, &.{ zig, "fetch", url });
+    };
+
+    if (hash_optional) |hash_specified| {
+        if (!std.mem.eql(u8, hash, hash_specified)) {
+            log.err(
+                \\bad hash
+                \\specified: {s}
+                \\fetched:   {s}
+                \\
+            , .{ hash_specified, hash });
+            return error.BadHash;
+        }
+    }
+
+    const cached = path_join(arena, &.{ global_cache, "p", hash, file_name });
+    errdefer log.err("copying from {s}", .{cached});
+
+    try std.fs.cwd().copyFile(cached, std.fs.cwd(), out, .{});
+}
+
+fn path_join(arena: Allocator, components: []const []const u8) []const u8 {
+    return std.fs.path.join(arena, components) catch |err| oom(err);
+}
+
+fn exec_ok(arena: Allocator, argv: []const []const u8) bool {
+    assert(argv.len > 0);
+    const result = std.process.Child.run(.{ .allocator = arena, .argv = argv }) catch return false;
+    return result.term == .Exited and result.term.Exited == 0;
+}
+
+fn exec(arena: Allocator, argv: []const []const u8) ![]const u8 {
+    assert(argv.len > 0);
+    const result = std.process.Child.run(.{ .allocator = arena, .argv = argv }) catch |err| {
+        log.err("running {s}: {}", .{ argv, err });
+        return err;
+    };
+    if (!(result.term == .Exited and result.term.Exited == 0)) {
+        log.err("running {s}: {}\n{s}", .{ argv, result.term, result.stderr });
+        return error.Exec;
+    }
+    if (std.mem.indexOfScalar(u8, result.stdout, '\n')) |first_newline| {
+        if (first_newline + 1 == result.stdout.len) {
+            return result.stdout[0 .. result.stdout.len - 1];
+        }
+    }
+    return result.stdout;
+}
+
+fn oom(_: error{OutOfMemory}) noreturn {
+    @panic("OOM");
+}

data/ext/tb_client/tigerbeetle/src/lsm/compaction.zig

@@ -255,17 +255,14 @@ pub fn ResourcePoolType(comptime Grid: type) type {
     };
 }
 
-pub fn CompactionType(
-    comptime Table: type,
-    comptime Tree: type,
-    comptime Storage: type,
-) type {
+pub fn CompactionType(comptime Tree: type, comptime Storage: type) type {
     return struct {
         const Compaction = @This();
 
         const Grid = GridType(Storage);
         const ResourcePool = ResourcePoolType(Grid);
 
+        const Table = Tree.Table;
         const Manifest = ManifestType(Table, Storage);
         const TableInfo = TableInfoType(Table);
         const TableInfoReference = Manifest.TableInfoReference;

data/ext/tb_client/tigerbeetle/src/lsm/composite_key.zig

@@ -37,9 +37,12 @@ pub fn CompositeKeyType(comptime Field: type) type {
 
         pub const Key = std.meta.Int(
             .unsigned,
+            // Little-endian:
             @bitSizeOf(u64) + @bitSizeOf(Field) + @bitSizeOf(Pad),
         );
 
+        // NB: ideally, we should swap `field` and `timestamp` here to  maintain
+        // little-endian order throughout, but we are constrained by existing data.
         field: Field align(field_bitsize_alignment),
         /// The most significant bit must be unset as it is used to indicate a tombstone.
         timestamp: u64,
@@ -65,7 +68,7 @@ pub fn CompositeKeyType(comptime Field: type) type {
                 return value.timestamp & ~tombstone_bit;
             } else {
                 comptime assert(@sizeOf(Key) == @sizeOf(Field) * 2);
-                return @as(Key, value.timestamp & ~tombstone_bit) | (@as(Key, value.field) << 64);
+                return (@as(Key, value.field) << 64) | @as(Key, value.timestamp & ~tombstone_bit);
             }
         }
 

data/ext/tb_client/tigerbeetle/src/lsm/forest.zig

@@ -241,6 +241,8 @@ pub fn ForestType(comptime _Storage: type, comptime groove_cfg: anytype) type {
         } = null,
 
         grid: *Grid,
+        next_tick: Grid.NextTick = undefined,
+
         grooves: Grooves,
         node_pool: NodePool,
         manifest_log: ManifestLog,
@@ -463,13 +465,24 @@ pub fn ForestType(comptime _Storage: type, comptime groove_cfg: anytype) type {
                 .manifest_log_done = false,
             };
 
-            // Manifest log compaction. Run on the last beat of each half-bar. Start before forest
-            // compaction for lesser fragmentation, as manifest log grid reservations are much
-            // smaller than compaction's.
-            // TODO: Figure out a plan wrt the pacing here. Putting it on the last beat kinda-sorta
-            // balances out, because we expect to naturally do less other compaction work on the
-            // last beat.
-            // The first bar has no manifest compaction.
+            // No compactions are run during the absolute first bar, or during
+            // the first bar of the checkpoint that we are currently recovering
+            // from (see `op_compacted` for the rationale behind this).
+            if (op < constants.lsm_compaction_ops or
+                forest.grid.superblock.working.vsr_state.op_compacted(op))
+            {
+                forest.compaction_progress.?.manifest_log_done = true;
+                forest.compaction_progress.?.trees_done = true;
+                forest.grid.on_next_tick(compact_finish_next_tick, &forest.next_tick);
+                return;
+            }
+
+            // Manifest log compaction. Run on the last beat of each half-bar.
+            // Start before forest compaction for lesser fragmentation, as
+            // manifest log grid reservations are much smaller than compaction's.
+            // TODO: Figure out a plan wrt the pacing here. Putting it on the
+            // last beat kinda-sorta balances out, because we expect to naturally
+            // do less other compaction work on the last beat.
             if (last_beat or last_half_beat) {
                 forest.manifest_log.compact(compact_manifest_log_callback, op);
             } else {
@@ -479,6 +492,13 @@ pub fn ForestType(comptime _Storage: type, comptime groove_cfg: anytype) type {
             forest.compaction_schedule.beat_start(compact_trees_callback, op);
         }
 
+        fn compact_finish_next_tick(next_tick: *Grid.NextTick) void {
+            const forest: *Forest = @alignCast(
+                @fieldParentPtr("next_tick", next_tick),
+            );
+            forest.compact_finish();
+        }
+
         fn compact_trees_callback(forest: *Forest) void {
             assert(forest.progress.? == .compact);
             assert(forest.compaction_progress != null);
@@ -547,7 +567,10 @@ pub fn ForestType(comptime _Storage: type, comptime groove_cfg: anytype) type {
             }
 
             if (last_beat or last_half_beat) {
-                assert(forest.compaction_schedule.bar_input_size == 0);
+                if (forest.compaction_schedule.bar_input_size) |bar_input_size| {
+                    assert(bar_input_size == 0);
+                    forest.compaction_schedule.bar_input_size = null;
+                }
 
                 // On the last beat of the bar, make sure that manifest log compaction is finished.
                 forest.manifest_log.compact_end();
@@ -891,8 +914,8 @@ fn CompactionScheduleType(comptime Forest: type, comptime Grid: type) type {
         pool: ResourcePool,
         next_tick: Grid.NextTick = undefined,
         callback: ?*const fn (*Forest) void = null,
-        bar_input_size: u64 = 0,
-        beat_input_size: u64 = 0,
+        bar_input_size: ?u64 = null,
+        beat_input_size: ?u64 = null,
 
         const CompactionSchedule = @This();
         const ResourcePool = ResourcePoolType(Grid);
@@ -926,17 +949,13 @@ fn CompactionScheduleType(comptime Forest: type, comptime Grid: type) type {
             assert(self.pool.grid_reservation == null);
 
             assert(self.callback == null);
-            assert(self.beat_input_size == 0);
+            assert(op >= constants.lsm_compaction_ops);
+            assert(!self.grid.superblock.working.vsr_state.op_compacted(op));
+            assert(self.beat_input_size == null);
 
+            self.beat_input_size = 0;
             self.callback = callback;
 
-            if (op < constants.lsm_compaction_ops or
-                self.grid.superblock.working.vsr_state.op_compacted(op))
-            {
-                self.beat_finish();
-                return;
-            }
-
             const half_bar = @divExact(constants.lsm_compaction_ops, 2);
             const compaction_beat = op % constants.lsm_compaction_ops;
 
@@ -945,8 +964,9 @@ fn CompactionScheduleType(comptime Forest: type, comptime Grid: type) type {
 
             if (first_beat or half_beat) {
                 assert(self.pool.blocks_acquired() == 0);
-                assert(self.bar_input_size == 0);
+                assert(self.bar_input_size == null);
 
+                var bar_input_size: u64 = 0;
                 for (0..constants.lsm_levels) |level_b| {
                     if (level_active(.{ .level_b = level_b, .op = op })) {
                         inline for (comptime std.enums.values(Forest.TreeID)) |tree_id| {
@@ -966,17 +986,18 @@ fn CompactionScheduleType(comptime Forest: type, comptime Grid: type) type {
                             );
                             const bar_input_values = compaction.bar_commence(op);
 
-                            self.bar_input_size += (bar_input_values * @sizeOf(Value));
+                            bar_input_size += (bar_input_values * @sizeOf(Value));
                         }
                     }
                 }
+                self.bar_input_size = bar_input_size;
             }
 
             const beats_total = half_bar;
             const beats_done = compaction_beat % half_bar;
             const beats_remaining = beats_total - beats_done;
 
-            self.beat_input_size = stdx.div_ceil(self.bar_input_size, beats_remaining);
+            self.beat_input_size = stdx.div_ceil(self.bar_input_size.?, beats_remaining);
 
             // This is akin to a dry run for the actual compaction work that is going to happen
             // during this beat, wherein we:
@@ -987,7 +1008,7 @@ fn CompactionScheduleType(comptime Forest: type, comptime Grid: type) type {
                 var beat_index_blocks_max: u64 = 1;
                 var beat_value_blocks_max: u64 = 1;
 
-                var beat_input_size = self.beat_input_size;
+                var beat_input_size = self.beat_input_size.?;
                 for (0..constants.lsm_levels) |level_b| {
                     if (level_active(.{ .level_b = level_b, .op = op })) {
                         inline for (comptime std.enums.values(Forest.TreeID)) |tree_id| {
@@ -1004,15 +1025,16 @@ fn CompactionScheduleType(comptime Forest: type, comptime Grid: type) type {
                             // The +1 is for imperfections in pacing our immutable table, which
                             // might cause us to overshoot by a single block (limited to 1 due
                             // to how the immutable table values are consumed.)
-                            beat_value_blocks_max += stdx.div_ceil(
+                            const beat_value_blocks = stdx.div_ceil(
                                 compaction.quotas.beat,
                                 Table.layout.block_value_count_max,
                             ) + 1;
-
-                            beat_index_blocks_max += stdx.div_ceil(
-                                beat_value_blocks_max,
+                            const beat_index_blocks = stdx.div_ceil(
+                                beat_value_blocks,
                                 Table.value_block_count_max,
                             );
+                            beat_value_blocks_max += beat_value_blocks;
+                            beat_index_blocks_max += beat_index_blocks;
 
                             beat_input_size -|= (compaction.quotas.beat * @sizeOf(Value));
                         }
@@ -1029,12 +1051,12 @@ fn CompactionScheduleType(comptime Forest: type, comptime Grid: type) type {
 
         fn beat_resume(self: *CompactionSchedule) void {
             assert(self.callback != null);
+            assert(self.pool.grid_reservation != null);
 
             if (self.beat_input_size == 0) {
                 self.beat_finish();
                 return;
             }
-            assert(self.pool.grid_reservation != null);
 
             const op = self.forest.progress.?.compact.op;
 
@@ -1065,8 +1087,8 @@ fn CompactionScheduleType(comptime Forest: type, comptime Grid: type) type {
                 inline else => |id| {
                     const Value = Forest.tree_info_for_id(id).Tree.Value;
                     const input_bytes_consumed = values_consumed * @sizeOf(Value);
-                    self.bar_input_size -= input_bytes_consumed;
-                    self.beat_input_size -|= input_bytes_consumed;
+                    self.bar_input_size.? -= input_bytes_consumed;
+                    self.beat_input_size.? -|= input_bytes_consumed;
                 },
             }
 
@@ -1075,11 +1097,16 @@ fn CompactionScheduleType(comptime Forest: type, comptime Grid: type) type {
 
         fn beat_finish(self: *CompactionSchedule) void {
             assert(self.callback != null);
+
+            assert(self.bar_input_size.? >= 0);
             assert(self.beat_input_size == 0);
+            self.beat_input_size = null;
+
             if (self.pool.grid_reservation) |reservation| {
                 self.grid.forfeit(reservation);
                 self.pool.grid_reservation = null;
             }
+
             self.grid.on_next_tick(beat_finish_next_tick, &self.next_tick);
         }
 

data/ext/tb_client/tigerbeetle/src/lsm/manifest_log.zig

@@ -676,6 +676,8 @@ pub fn ManifestLogType(comptime Storage: type) type {
             assert(manifest_log.blocks.count ==
                 manifest_log.blocks_closed + @intFromBool(manifest_log.entry_count > 0));
             assert(manifest_log.compact_blocks == null);
+            assert(op >= constants.lsm_compaction_ops);
+            assert(!manifest_log.superblock.working.vsr_state.op_compacted(op));
 
             // TODO: Currently manifest compaction is hardcoded to run on the last beat of each
             // half-bar.
@@ -685,14 +687,6 @@ pub fn ManifestLogType(comptime Storage: type) type {
 
             manifest_log.grid.trace.start(.compact_manifest);
 
-            if (op < constants.lsm_compaction_ops or
-                manifest_log.superblock.working.vsr_state.op_compacted(op))
-            {
-                manifest_log.read_callback = callback;
-                manifest_log.grid.on_next_tick(compact_tick_callback, &manifest_log.next_tick);
-                return;
-            }
-
             manifest_log.compact_blocks = @min(
                 manifest_log.pace.half_bar_compact_blocks(.{
                     .log_blocks_count = @intCast(manifest_log.log_block_checksums.count),
@@ -712,22 +706,6 @@ pub fn ManifestLogType(comptime Storage: type) type {
             manifest_log.flush(compact_next_block);
         }
 
-        fn compact_tick_callback(next_tick: *Grid.NextTick) void {
-            const manifest_log: *ManifestLog = @alignCast(@fieldParentPtr("next_tick", next_tick));
-            assert(manifest_log.write_callback == null);
-            assert(manifest_log.grid_reservation == null);
-            assert(manifest_log.blocks_closed == 0);
-            assert(manifest_log.blocks.count == 0);
-            assert(manifest_log.entry_count == 0);
-            assert(manifest_log.compact_blocks == null);
-
-            manifest_log.grid.trace.stop(.compact_manifest);
-
-            const callback = manifest_log.read_callback.?;
-            manifest_log.read_callback = null;
-            callback(manifest_log);
-        }
-
         fn compact_next_block(manifest_log: *ManifestLog) void {
             assert(manifest_log.opened);
             assert(!manifest_log.reading);

data/ext/tb_client/tigerbeetle/src/lsm/tree.zig

@@ -54,7 +54,7 @@ pub fn TreeType(comptime TreeTable: type, comptime Storage: type) type {
         const KeyRange = Manifest.KeyRange;
 
         const CompactionType = @import("compaction.zig").CompactionType;
-        pub const Compaction = CompactionType(Table, Tree, Storage);
+        pub const Compaction = CompactionType(Tree, Storage);
 
         pub const LookupMemoryResult = union(enum) {
             negative,
@@ -218,13 +218,12 @@ pub fn TreeType(comptime TreeTable: type, comptime Storage: type) type {
         ///
         /// Specifically, it checks whether the key exists within the Tree's key range.
         pub fn key_range_contains(tree: *const Tree, snapshot: u64, key: Key) bool {
-            if (snapshot == snapshot_latest) {
-                return tree.key_range != null and
-                    tree.key_range.?.key_min <= key and
-                    key <= tree.key_range.?.key_max;
-            } else {
-                return true;
-            }
+            // TODO We currently assume that the snapshot passed in is the latest snapshot.
+            // This must be changed when persistent snapshots are implemented.
+            assert(snapshot < snapshot_latest);
+            return tree.key_range != null and
+                tree.key_range.?.key_min <= key and
+                key <= tree.key_range.?.key_max;
         }
 
         /// This function is intended to never be called by regular code. It only

data/ext/tb_client/tigerbeetle/src/message_bus.zig

@@ -717,6 +717,10 @@ pub fn MessageBusType(comptime IO: type) type {
                     // * client_likely  → client
                     assert(connection.peer == .unknown or connection.peer == .client_likely);
 
+                    if (connection.peer == .client_likely) {
+                        assert(connection.peer.client_likely == client_id);
+                    }
+
                     // If there is a connection to this client, terminate and replace it.
                     const result = bus.clients.getOrPutAssumeCapacity(client_id);
                     if (result.found_existing) {
@@ -737,7 +741,7 @@ pub fn MessageBusType(comptime IO: type) type {
                     }
 
                     result.value_ptr.* = connection;
-                    log.info("{}: set_and_verify_peer connection from client={}", .{
+                    log.info("{}: set_and_verify_peer: connection from client={}", .{
                         bus.id,
                         client_id,
                     });
@@ -756,7 +760,7 @@ pub fn MessageBusType(comptime IO: type) type {
                                 bus.clients.getOrPutAssumeCapacity(client_id);
                             if (!result.found_existing) {
                                 result.value_ptr.* = connection;
-                                log.info("{}: set_and_verify_peer connection from " ++
+                                log.info("{}: set_and_verify_peer: connection from " ++
                                     "client_likely={}", .{ bus.id, client_id });
                             }
                         },
@@ -771,7 +775,7 @@ pub fn MessageBusType(comptime IO: type) type {
             return true;
         }
 
-        /// Attempt moving messages from recv buffer into replcia for processing. Called when recv
+        /// Attempt moving messages from recv buffer into replica for processing. Called when recv
         /// syscall completes, or when a replica signals readiness to consume previously suspended
         /// messages.
         fn recv_buffer_drain(bus: *MessageBus, connection: *Connection) void {
@@ -1065,6 +1069,7 @@ pub fn MessageBusType(comptime IO: type) type {
             assert(connection.recv_buffer == null);
             assert(connection.send_queue.empty());
             assert(connection.fd == null);
+            assert(!bus.connections_suspended.contains(connection));
 
             result catch |err| {
                 log.warn("{}: on_close: to={} {}", .{ bus.id, connection.peer, err });