rubydb 0.1.4 → 0.1.6

data/scripts/replication_network_failover_drill

@@ -1,255 +1,287 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-# Exercises the real replication sockets through a fault-injecting TCP proxy.
-# The proxy is deliberately small and local: it drops the live stream to model
-# a network partition while both database processes remain alive. This makes
-# the scenario reproducible on developer machines and in CI without changing
-# production replication code or requiring host firewall privileges.
-
-require "json"
-require "socket"
-require "stringio"
-require "tmpdir"
-require "timeout"
-
-root = File.expand_path("..", __dir__)
-$LOAD_PATH.unshift(File.join(root, "lib"))
-require "rubydb"
-
-TABLE = "network_failover_events"
-COLUMNS = [RubyDB::Catalog::Column.new(:id, :integer, primary_key: true, null: false)].freeze
-
-class TcpPartitionProxy
-  attr_reader :port
-
-  def initialize(target_host, target_port)
-    @target_host = target_host
-    @target_port = target_port
-    @server = TCPServer.new("127.0.0.1", 0)
-    @port = @server.addr[1]
-    @lock = Mutex.new
-    @pairs = []
-    @partitioned = false
-    @running = false
-  end
-
-  def start
-    @lock.synchronize { @running = true }
-    @thread = Thread.new { accept_loop }
-    self
-  end
-
-  def partition!
-    pairs = @lock.synchronize do
-      @partitioned = true
-      current = @pairs.dup
-      @pairs.clear
-      current
-    end
-    pairs.each { |client, target| close_pair(client, target) }
-    true
-  end
-
-  def heal!
-    @lock.synchronize { @partitioned = false }
-    true
-  end
-
-  def stop
-    pairs = @lock.synchronize do
-      @running = false
-      current = @pairs.dup
-      @pairs.clear
-      current
-    end
-    @server.close rescue nil
-    pairs.each { |client, target| close_pair(client, target) }
-    @thread&.join(2)
-    @thread&.kill if @thread&.alive?
-    true
-  end
-
-  private
-
-  def accept_loop
-    while @lock.synchronize { @running }
-      client = @server.accept
-      partitioned = @lock.synchronize { @partitioned }
-      if partitioned
-        client.close
-        next
-      end
-
-      target = TCPSocket.new(@target_host, @target_port)
-      pair = [client, target]
-      @lock.synchronize { @pairs << pair }
-      Thread.new { forward(pair, client, target) }
-      Thread.new { forward(pair, target, client) }
-    end
-  rescue IOError, Errno::EBADF
-    # Expected when stop closes the listener.
-  rescue StandardError
-    retry if @lock.synchronize { @running }
-  end
-
-  def forward(pair, source, destination)
-    loop do
-      ready = IO.select([source], nil, nil, 0.2)
-      next unless ready
-
-      destination.write(source.readpartial(16_384))
-      destination.flush
-    end
-  rescue EOFError, IOError, SystemCallError
-    # A dropped stream is the modeled partition.
-  ensure
-    @lock.synchronize { @pairs.delete(pair) }
-    close_pair(*pair)
-  end
-
-  def close_pair(first, second)
-    first.close rescue nil
-    second.close rescue nil
-  end
-end
-
-def available_port
-  probe = TCPServer.new("127.0.0.1", 0)
-  probe.addr[1]
-ensure
-  probe&.close
-end
-
-def wait_until(timeout: 10)
-  Timeout.timeout(timeout) do
-    loop do
-      return true if yield
-      sleep 0.02
-    end
-  end
-end
-
-def insert(engine, id)
-  engine.begin_transaction
-  engine.insert_row(TABLE, COLUMNS, id: id)
-  engine.commit_transaction
-rescue StandardError
-  engine.rollback_transaction rescue nil
-  raise
-end
-
-def rows(engine)
-  return [] unless engine.table_exists?(TABLE)
-
-  engine.select_rows(TABLE, COLUMNS).map { |row| (row[:id] || row["id"]).to_i }.sort
-end
-
-result = nil
-original_stdout = $stdout
-$stdout = StringIO.new
-begin
-  Dir.mktmpdir("rubydb-network-failover-") do |directory|
-  primary_engine = RubyDB::Storage::Engine.new(File.join(directory, "primary.rdb"), auto_cleanup: false, auto_vacuum: false)
-  replica_engine = RubyDB::Storage::Engine.new(File.join(directory, "replica.rdb"), auto_cleanup: false, auto_vacuum: false)
-  primary = replica = promoted = proxy = nil
-  fence_path = File.join(directory, "cluster.fence")
-
-  primary_engine.create_table(TABLE, COLUMNS)
-  primary_port = available_port
-  proxy = TcpPartitionProxy.new("127.0.0.1", primary_port).start
-
-  begin
-    primary = RubyDB::Replication::Primary.new(
-      primary_engine,
-      host: "127.0.0.1",
-      replication_port: primary_port,
-      log_dir: File.join(directory, "primary-log"),
-      fence_path: fence_path,
-      heartbeat_interval: 0.05,
-      replication_timeout: 0.5,
-      node_id: "network-primary"
-    )
-    replica = RubyDB::Replication::Replica.new(
-      replica_engine,
-      primary_host: "127.0.0.1",
-      replication_port: proxy.port,
-      retry_interval: 0.02,
-      max_retry_attempts: 500,
-      heartbeat_interval: 0.05,
-      state_path: File.join(directory, "replica-state.json")
-    )
-    primary.start
-    replica.start
-  end
-
-  wait_until { replica.replication_status[:state].to_s == "streaming" }
-  insert(primary_engine, 1)
-  wait_until { rows(replica_engine) == [1] }
-
-  proxy.partition!
-  wait_until { %w[disconnected failed].include?(replica.replication_status[:state].to_s) }
-  insert(primary_engine, 2)
-  raise "partition did not isolate replica" unless rows(replica_engine) == [1]
-
-  proxy.heal!
-  wait_until { replica.replication_status[:state].to_s == "streaming" }
-  wait_until { rows(replica_engine) == [1, 2] }
-
-  # Partition a synchronized stream, fence the old writer, and promote only
-  # the replica that is known to have replayed through the recovery point.
-  proxy.partition!
-  wait_until { %w[disconnected failed].include?(replica.replication_status[:state].to_s) }
-  # Put the replica into the stable disconnected state before promotion. A
-  # continuously retrying, failed connection is not itself proof of a safe
-  # promotion even when its replay positions match.
-  replica.stop
-  RubyDB::Replication::FencingLease.new(fence_path, "external-fencer").acquire!
-  old_primary_rejected = false
-  begin
-    insert(primary_engine, 3)
-  rescue RubyDB::ReplicationError
-    old_primary_rejected = true
-  end
-  raise "stale primary accepted a write" unless old_primary_rejected
-
-  replica.promote_to_primary(recovery_point: 2)
-  promoted_port = available_port
-  begin
-    promoted = RubyDB::Replication::Primary.new(
-      replica_engine,
-      host: "127.0.0.1",
-      replication_port: promoted_port,
-      log_dir: File.join(directory, "promoted-log"),
-      fence_path: fence_path,
-      heartbeat_interval: 0.05,
-      replication_timeout: 0.5,
-      node_id: "promoted-primary"
-    )
-    promoted.start
-  end
-  insert(replica_engine, 3)
-  raise "promoted primary did not accept a write" unless rows(replica_engine) == [1, 2, 3]
-
-  result = {
-    success: true,
-    network_partition: true,
-    partitioned_primary_stayed_running: true,
-    replica_caught_up_after_heal: true,
-    stale_primary_rejected: old_primary_rejected,
-    promoted_rows: rows(replica_engine),
-    fencing_epoch_after_promotion: promoted.fencing_status[:epoch]
-  }
-ensure
-  promoted&.stop rescue nil
-  primary&.stop rescue nil
-  replica&.stop rescue nil
-  proxy&.stop rescue nil
-  replica_engine&.close if replica_engine&.open?
-  primary_engine&.close if primary_engine&.open?
-  end
-ensure
-  $stdout = original_stdout
-end
-
-puts JSON.generate(result) if result
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Exercises the real replication sockets through a fault-injecting TCP proxy.
+# The proxy is deliberately small and local: it drops the live stream to model
+# a network partition while both database processes remain alive. This makes
+# the scenario reproducible on developer machines and in CI without changing
+# production replication code or requiring host firewall privileges.
+
+require "json"
+require "socket"
+require "stringio"
+require "tmpdir"
+require "timeout"
+
+root = File.expand_path("..", __dir__)
+$LOAD_PATH.unshift(File.join(root, "lib"))
+require "rubydb"
+
+TABLE = "network_failover_events"
+COLUMNS = [RubyDB::Catalog::Column.new(:id, :integer, primary_key: true, null: false)].freeze
+
+class TcpPartitionProxy
+  attr_reader :port
+
+  def initialize(target_host, target_port)
+    @target_host = target_host
+    @target_port = target_port
+    @server = TCPServer.new("127.0.0.1", 0)
+    @port = @server.addr[1]
+    @lock = Mutex.new
+    @pairs = []
+    @partitioned = false
+    @running = false
+  end
+
+  def start
+    @lock.synchronize { @running = true }
+    @thread = Thread.new { accept_loop }
+    self
+  end
+
+  def partition!
+    pairs = @lock.synchronize do
+      @partitioned = true
+      current = @pairs.dup
+      @pairs.clear
+      current
+    end
+    pairs.each { |client, target| close_pair(client, target) }
+    true
+  end
+
+  def heal!
+    @lock.synchronize { @partitioned = false }
+    true
+  end
+
+  def stop
+    pairs = @lock.synchronize do
+      @running = false
+      current = @pairs.dup
+      @pairs.clear
+      current
+    end
+    begin
+      @server.close
+    rescue
+      nil
+    end
+    pairs.each { |client, target| close_pair(client, target) }
+    @thread&.join(2)
+    @thread&.kill if @thread&.alive?
+    true
+  end
+
+  private
+
+  def accept_loop
+    while @lock.synchronize { @running }
+      client = @server.accept
+      partitioned = @lock.synchronize { @partitioned }
+      if partitioned
+        client.close
+        next
+      end
+
+      target = TCPSocket.new(@target_host, @target_port)
+      pair = [client, target]
+      @lock.synchronize { @pairs << pair }
+      Thread.new { forward(pair, client, target) }
+      Thread.new { forward(pair, target, client) }
+    end
+  rescue IOError, Errno::EBADF
+    # Expected when stop closes the listener.
+  rescue
+    retry if @lock.synchronize { @running }
+  end
+
+  def forward(pair, source, destination)
+    loop do
+      ready = IO.select([source], nil, nil, 0.2)
+      next unless ready
+
+      destination.write(source.readpartial(16_384))
+      destination.flush
+    end
+  rescue IOError, SystemCallError
+    # A dropped stream is the modeled partition.
+  ensure
+    @lock.synchronize { @pairs.delete(pair) }
+    close_pair(*pair)
+  end
+
+  def close_pair(first, second)
+    begin
+      first.close
+    rescue
+      nil
+    end
+    begin
+      second.close
+    rescue
+      nil
+    end
+  end
+end
+
+def available_port
+  probe = TCPServer.new("127.0.0.1", 0)
+  probe.addr[1]
+ensure
+  probe&.close
+end
+
+def wait_until(timeout: 10)
+  Timeout.timeout(timeout) do
+    loop do
+      return true if yield
+      sleep 0.02
+    end
+  end
+end
+
+def insert(engine, id)
+  engine.begin_transaction
+  engine.insert_row(TABLE, COLUMNS, id: id)
+  engine.commit_transaction
+rescue
+  begin
+    engine.rollback_transaction
+  rescue
+    nil
+  end
+  raise
+end
+
+def rows(engine)
+  return [] unless engine.table_exists?(TABLE)
+
+  engine.select_rows(TABLE, COLUMNS).map { |row| (row[:id] || row["id"]).to_i }.sort
+end
+
+result = nil
+original_stdout = $stdout
+$stdout = StringIO.new
+begin
+  Dir.mktmpdir("rubydb-network-failover-") do |directory|
+    primary_engine = RubyDB::Storage::Engine.new(File.join(directory, "primary.rdb"), auto_cleanup: false, auto_vacuum: false)
+    replica_engine = RubyDB::Storage::Engine.new(File.join(directory, "replica.rdb"), auto_cleanup: false, auto_vacuum: false)
+    primary = replica = promoted = proxy = nil
+    fence_path = File.join(directory, "cluster.fence")
+
+    primary_engine.create_table(TABLE, COLUMNS)
+    primary_port = available_port
+    proxy = TcpPartitionProxy.new("127.0.0.1", primary_port).start
+
+    begin
+      primary = RubyDB::Replication::Primary.new(
+        primary_engine,
+        host: "127.0.0.1",
+        replication_port: primary_port,
+        log_dir: File.join(directory, "primary-log"),
+        fence_path: fence_path,
+        heartbeat_interval: 0.05,
+        replication_timeout: 0.5,
+        node_id: "network-primary"
+      )
+      replica = RubyDB::Replication::Replica.new(
+        replica_engine,
+        primary_host: "127.0.0.1",
+        replication_port: proxy.port,
+        retry_interval: 0.02,
+        max_retry_attempts: 500,
+        heartbeat_interval: 0.05,
+        state_path: File.join(directory, "replica-state.json")
+      )
+      primary.start
+      replica.start
+    end
+
+    wait_until { replica.replication_status[:state].to_s == "streaming" }
+    insert(primary_engine, 1)
+    wait_until { rows(replica_engine) == [1] }
+
+    proxy.partition!
+    wait_until { %w[disconnected failed].include?(replica.replication_status[:state].to_s) }
+    insert(primary_engine, 2)
+    raise "partition did not isolate replica" unless rows(replica_engine) == [1]
+
+    proxy.heal!
+    wait_until { replica.replication_status[:state].to_s == "streaming" }
+    wait_until { rows(replica_engine) == [1, 2] }
+
+    # Partition a synchronized stream, fence the old writer, and promote only
+    # the replica that is known to have replayed through the recovery point.
+    proxy.partition!
+    wait_until { %w[disconnected failed].include?(replica.replication_status[:state].to_s) }
+    # Put the replica into the stable disconnected state before promotion. A
+    # continuously retrying, failed connection is not itself proof of a safe
+    # promotion even when its replay positions match.
+    replica.stop
+    RubyDB::Replication::FencingLease.new(fence_path, "external-fencer").acquire!
+    old_primary_rejected = false
+    begin
+      insert(primary_engine, 3)
+    rescue RubyDB::ReplicationError
+      old_primary_rejected = true
+    end
+    raise "stale primary accepted a write" unless old_primary_rejected
+
+    replica.promote_to_primary(recovery_point: 2)
+    promoted_port = available_port
+    begin
+      promoted = RubyDB::Replication::Primary.new(
+        replica_engine,
+        host: "127.0.0.1",
+        replication_port: promoted_port,
+        log_dir: File.join(directory, "promoted-log"),
+        fence_path: fence_path,
+        heartbeat_interval: 0.05,
+        replication_timeout: 0.5,
+        node_id: "promoted-primary"
+      )
+      promoted.start
+    end
+    insert(replica_engine, 3)
+    raise "promoted primary did not accept a write" unless rows(replica_engine) == [1, 2, 3]
+
+    result = {
+      success: true,
+      network_partition: true,
+      partitioned_primary_stayed_running: true,
+      replica_caught_up_after_heal: true,
+      stale_primary_rejected: old_primary_rejected,
+      promoted_rows: rows(replica_engine),
+      fencing_epoch_after_promotion: promoted.fencing_status[:epoch]
+    }
+  ensure
+    begin
+      promoted&.stop
+    rescue
+      nil
+    end
+    begin
+      primary&.stop
+    rescue
+      nil
+    end
+    begin
+      replica&.stop
+    rescue
+      nil
+    end
+    begin
+      proxy&.stop
+    rescue
+      nil
+    end
+    replica_engine&.close if replica_engine&.open?
+    primary_engine&.close if primary_engine&.open?
+  end
+ensure
+  $stdout = original_stdout
+end
+
+puts JSON.generate(result) if result

data/scripts/restore_drill

@@ -1,45 +1,45 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-# End-to-end backup/restore drill. This intentionally uses a fresh destination
-# and opens the restored files with the real storage engine.
-
-require "tmpdir"
-
-root = File.expand_path("..", __dir__)
-$LOAD_PATH.unshift(File.join(root, "lib"))
-require "rubydb"
-
-Dir.mktmpdir("rubydb-restore-drill-") do |directory|
-  source = File.join(directory, "source", "rubydb.rdb")
-  backup_dir = File.join(directory, "backups")
-  verification_dir = File.join(directory, "verification")
-  restore_dir = File.join(directory, "restored")
-  columns = [RubyDB::Catalog::Column.new("id", :integer, primary_key: true, null: false)]
-
-  engine = RubyDB::Storage::Engine.new(source, auto_cleanup: false, auto_vacuum: false)
-  engine.create_table("restore_drill", columns)
-  engine.insert_row("restore_drill", columns, [1])
-  engine.insert_row("restore_drill", columns, [2])
-
-  backup = RubyDB::Backup::Backup.new(engine, backup_dir: backup_dir, include_schema: true)
-  created = backup.create_backup
-  abort "backup failed: #{created[:error]}" unless created[:success]
-
-  verification = RubyDB::Backup::Verification.new(verification_dir: verification_dir)
-  result = verification.verify_restore(created[:backup_path])
-  abort "restore verification failed: #{result[:error]}" unless result[:success]
-  abort "restore row-count mismatch" unless result[:row_counts]["restore_drill"] == 2
-
-  restored = RubyDB::Backup::Restore.new(nil, backup_dir: backup_dir).restore(
-    created[:backup_path], destination: restore_dir
-  )
-  abort "restore failed: #{restored[:error]}" unless restored[:success]
-  restored_engine = RubyDB::Storage::Engine.new(File.join(restore_dir, "rubydb.rdb"), auto_cleanup: false, auto_vacuum: false)
-  abort "restored data mismatch" unless restored_engine.table_row_count("restore_drill") == 2
-
-  puts "Restore drill passed: #{created[:backup_name]} (2 rows restored and reopened)"
-ensure
-  restored_engine&.close
-  engine&.close
-end
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# End-to-end backup/restore drill. This intentionally uses a fresh destination
+# and opens the restored files with the real storage engine.
+
+require "tmpdir"
+
+root = File.expand_path("..", __dir__)
+$LOAD_PATH.unshift(File.join(root, "lib"))
+require "rubydb"
+
+Dir.mktmpdir("rubydb-restore-drill-") do |directory|
+  source = File.join(directory, "source", "rubydb.rdb")
+  backup_dir = File.join(directory, "backups")
+  verification_dir = File.join(directory, "verification")
+  restore_dir = File.join(directory, "restored")
+  columns = [RubyDB::Catalog::Column.new("id", :integer, primary_key: true, null: false)]
+
+  engine = RubyDB::Storage::Engine.new(source, auto_cleanup: false, auto_vacuum: false)
+  engine.create_table("restore_drill", columns)
+  engine.insert_row("restore_drill", columns, [1])
+  engine.insert_row("restore_drill", columns, [2])
+
+  backup = RubyDB::Backup::Backup.new(engine, backup_dir: backup_dir, include_schema: true)
+  created = backup.create_backup
+  abort "backup failed: #{created[:error]}" unless created[:success]
+
+  verification = RubyDB::Backup::Verification.new(verification_dir: verification_dir)
+  result = verification.verify_restore(created[:backup_path])
+  abort "restore verification failed: #{result[:error]}" unless result[:success]
+  abort "restore row-count mismatch" unless result[:row_counts]["restore_drill"] == 2
+
+  restored = RubyDB::Backup::Restore.new(nil, backup_dir: backup_dir).restore(
+    created[:backup_path], destination: restore_dir
+  )
+  abort "restore failed: #{restored[:error]}" unless restored[:success]
+  restored_engine = RubyDB::Storage::Engine.new(File.join(restore_dir, "rubydb.rdb"), auto_cleanup: false, auto_vacuum: false)
+  abort "restored data mismatch" unless restored_engine.table_row_count("restore_drill") == 2
+
+  puts "Restore drill passed: #{created[:backup_name]} (2 rows restored and reopened)"
+ensure
+  restored_engine&.close
+  engine&.close
+end