evinrude 0.0.1

data/smoke_tests/005_sleepy_master.rb

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+
+require_relative "./smoke_test_helper"
+
+require "evinrude"
+require "logger"
+
+Thread.current.name = "MT"
+
+nodes = spawn_nodes(5)
+
+wait_for_stability(nodes)
+
+leader = nodes.find { |n| n.c.leader? }
+followers = nodes.select { |n| n.c.follower? }
+
+leader.c.singleton_class.prepend(FaultInjector)
+leader.c.pause!(:issue_append_entries)
+
+until new_leader = followers.find { |n| n.c.leader? }
+	leader.t.join(1)
+end
+
+leader.c.unpause!(:issue_append_entries)
+
+leader.t.join(1)
+
+assert leader.c.follower?

data/smoke_tests/006_join_via_follower.rb

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+
+require_relative "./smoke_test_helper"
+
+require "evinrude"
+
+Thread.current.name = "MT"
+
+nodes = spawn_nodes(3)
+
+wait_for_stability(nodes)
+wait_for_consensus(nodes)
+
+follower = nodes.find { |n| n.c.follower? }
+
+newbie = ClusterNode.new(join_hints: [{ address: follower.c.address, port: follower.c.port }], shared_keys: ["s3kr1t"], logger: default_logger)
+newbie.t.name = "NW"
+
+newbie.t.join(0.1)
+
+until newbie.c.follower?
+	newbie.t.join(0.1)
+	logger.info(logloc) { "Waiting for newbie to become a follower" }
+end
+
+assert newbie.c.follower?, "Newbie has joined the cluster"

data/smoke_tests/007_snapshot_madness.rb

@@ -0,0 +1,97 @@
+#!/usr/bin/env ruby
+
+require_relative "./smoke_test_helper"
+
+require "evinrude"
+
+Thread.current.name = "MT"
+
+nodes = spawn_nodes(3, storage_base_dir: $tmpdir.to_s)
+
+wait_for_stability(nodes)
+wait_for_consensus(nodes)
+
+nodes.each.with_index { |n, i| n.c.command(n.t.name) }
+
+wait_for_consensus(nodes)
+
+first_state = nodes.first.c.state
+
+nodes.each do |n|
+	assert_equal YAML.load_stream($tmpdir.join(n.t.name, "log.yaml").read).last[:commit_entries_to], [n.c.instance_variable_get(:@commit_index)], "#{n.t.name} logged commit_entries_to"
+end
+
+nodes.each { |n| n.c.__send__(:take_snapshot) }
+
+nodes.each do |n|
+	snap = YAML.load_file($tmpdir.join(n.t.name, "snapshot.yaml"))
+
+	assert_equal snap.state, first_state, "#{n.t.name} state"
+end
+
+nodes.each.with_index { |n, i| n.c.command(n.t.name) }
+
+wait_for_consensus(nodes)
+
+mid_state = nodes.first.c.state
+
+victim = nodes.find { |n| n.t.name == mid_state }
+nodes.delete(victim)
+
+victim.t.kill
+
+assert_equal YAML.load_stream($tmpdir.join(victim.t.name, "log.yaml").read).last[:commit_entries_to], [nodes.first.c.instance_variable_get(:@commit_index)], "Victim #{victim.t.name} had up-to-date commit_index"
+
+nodes.each.with_index { |n, i| n.c.command(n.t.name) }
+
+wait_for_consensus(nodes)
+
+new_state = nodes.first.c.state
+
+assert new_state != mid_state, "Check that we've got a new state"
+
+lazarus = ClusterNode.new(shared_keys: ["s3kr1t"], logger: default_logger, storage_dir: $tmpdir.join(victim.t.name).to_s)
+lazarus.t.name = "LZ"
+
+nodes << lazarus
+
+wait_for_consensus(nodes)
+
+assert_equal lazarus.c.state, new_state, "Ensure lazarus has synced up"
+
+# Phatten the log file
+log_file = $tmpdir.join("C1", "log.yaml")
+until log_file.size > 500_000
+	lazarus.c.command("logloglog!" * 1000)
+end
+
+# Now keep phattening the log file until it gets tiny again -- indicating
+# a snapshot was taken -- or it gets stupidly huge
+until log_file.size < 500_000 || log_file.size > 2_000_000
+	lazarus.c.command("xyzzy!" * rand(1000))
+end
+
+assert log_file.size < 32_000_000, "Log file didn't get truncated; was a snapshot even taken?"
+
+# Can a new node, starting completely from scratch, load from a master's snapshot?
+newbie = ClusterNode.new(shared_keys: ["s3kr1t"], logger: default_logger, storage_dir: $tmpdir.join("newbie").to_s, node_name: "NB", join_hints: [{ address: lazarus.c.address, port: lazarus.c.port }])
+newbie.t.name = "NB"
+
+nodes << newbie
+
+wait_for_consensus(nodes)
+
+assert_equal newbie.c.state, lazarus.c.state, "Ensure newbie is synced"
+
+# As a final test, let's make sure the in-memory log trimming is working OK
+nodes.each { |n| n.t.kill }
+
+log = nodes.first.c.instance_variable_get(:@log)
+
+2000.times do |i|
+	log.append(Evinrude::LogEntry::Null.new(term: i + 1))
+end
+
+assert log.instance_variable_get(:@entries).length < 1001, "Log entries length kept under control"
+assert_equal log.last_entry_term, 2000, "All log entries are recorded"
+assert_equal log.instance_variable_get(:@snapshot_last_term), log.instance_variable_get(:@entries).first.term - 1, "The snapshot->live log entries transition is correct"

data/smoke_tests/008_downsizing.rb

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+
+require_relative "./smoke_test_helper"
+
+require "evinrude"
+
+Thread.current.name = "MT"
+
+nodes = spawn_nodes(5)
+
+wait_for_stability(nodes)
+wait_for_consensus(nodes)
+
+nodes.first.c.command("famous five")
+
+nodes.each { |n| assert_equal n.c.state, "famous five", n.t.name }
+
+2.times do
+	victim = nodes.last
+
+	info = Evinrude::NodeInfo.new(address: victim.c.address, port: victim.c.port, name: victim.c.node_name)
+
+	default_logger.info(logloc) { "Removing node #{victim.t.name}" }
+	victim.c.remove_node(info)
+	victim.t.kill
+
+	nodes.delete(victim)
+end
+
+# We should still have a working cluster at the moment, regardless
+nodes.first.c.command("now we are three")
+
+nodes.each { |n| assert_equal n.c.state, "now we are three", n.t.name }
+
+victim = nodes.first
+default_logger.info(logloc) { "Crashing node #{victim.t.name}" }
+victim.t.kill
+nodes.delete(victim)
+
+# If the earlier removals went OK, we should *still* have a working cluster
+nodes.first.c.command("two is tops")
+
+nodes.each { |n| assert_equal n.c.state, "two is tops", n.t.name }

data/smoke_tests/009_disaster_recovery.rb

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+
+require_relative "./smoke_test_helper"
+
+require "evinrude"
+
+Thread.current.name = "MT"
+
+nodes = spawn_nodes(5)
+
+wait_for_stability(nodes)
+wait_for_consensus(nodes)
+
+nodes.first.c.command("famous five")
+
+nodes.each { |n| assert_equal n.c.state, "famous five", n.t.name }
+
+# For maximum confusion, we make sure that we crash the master
+victim = nodes.find { |n| n.c.leader? }
+default_logger.info(logloc) { "Crashing node #{victim.t.name}" }
+victim.t.kill
+nodes.delete(victim)
+
+crashed_nodes_info = []
+
+# And take out another two nodes, to make sure we've lost consensus, but we'll
+# keep a note of their identity for later consumption
+2.times do
+	victim = nodes.last
+
+	crashed_nodes_info << victim.c.node_info
+	default_logger.info(logloc) { "Crashing node #{victim.t.name}" }
+	victim.t.kill
+
+	nodes.delete(victim)
+end
+
+crashed_nodes_info.each do |ni|
+	nodes.first.c.remove_node(ni, unsafe: true)
+end
+
+# This should work... eventually
+
+nodes.last.c.command("survival of the fittest")
+
+nodes.each { |n| assert_equal n.c.state, "survival of the fittest", n.t.name }

data/smoke_tests/999_final_smoke_test.rb

@@ -0,0 +1,279 @@
+require_relative "./smoke_test_helper"
+
+Thread.current.name = "FST"
+
+class FinalSmokeTest
+	attr_reader :op_log, :m
+
+	def initialize(seed:, interval:, op_count:, logger:, edn_file: File.open("/dev/null"))
+		@seed, @interval, @op_count, @logger, @edn_file = seed, interval, op_count, logger, edn_file
+
+		@rng = Random.new(@seed)
+		@op_log = []
+		@olm = Mutex.new
+		@m = Mutex.new
+		@ednm = Mutex.new
+	end
+
+	def run(nodes)
+		threads = []
+
+		@edn_file.write("[")
+
+		@op_count.times do |i|
+			start = Time.now
+
+			op = select_op
+
+			o = op.new(nodes: nodes, rng: @rng, logger: @logger, op_id: i, fst: self)
+
+			threads << Thread.new do
+				Thread.current.name = "O#{i}"
+
+				o.apply
+			end
+
+			sleep [Time.now - start + @interval.rand, 0].max
+		end
+
+		threads.each(&:join)
+
+		print "..."
+
+=begin
+		until threads.empty?
+			threads.select!(&:alive?)
+
+			threads.each do |th|
+				th.join(1)
+				if th.status
+					puts
+					puts
+					puts "{#{th.name})"
+					puts th.backtrace
+					th.kill
+				end
+			end
+		end
+=end
+	ensure
+		@edn_file.puts("]")
+	end
+
+	def <<(v)
+		@olm.synchronize { @op_log << v }
+	end
+
+	def edn(e)
+		@ednm.synchronize do
+			pid = if e[:process] =~ /\AC(\d+)(?:-(\d+))?\z/
+				r = $1.to_i
+				m = $2.to_i
+				m * 5 + r
+			else
+				raise "FFS"
+			end
+
+			@edn_file.puts "{:process #{pid}, :type #{e[:type].inspect}, :f #{e[:f].inspect}, :value #{e[:value].inspect}}"
+			@edn_file.fsync
+		end
+	end
+
+
+	private
+
+	def select_op
+		n = @rng.rand(op_weight_count)
+
+		Op.const_get(Op.constants.sort.find { |c| n -= Op.const_get(c)::WEIGHT; n <= 0 })
+	end
+
+	def op_weight_count
+		@op_weight_count ||= Op.constants.inject(0) { |a, c| a + Op.const_get(c)::WEIGHT }
+	end
+
+	module EvinrudeHooks
+		def initialize(*_)
+			super
+		end
+	end
+
+	class Op
+		include Evinrude::LoggingHelpers
+
+		def initialize(nodes:, rng:, logger:, op_id:, fst:)
+			@nodes, @rng, @logger, @op_id, @fst = nodes, rng, logger, op_id, fst
+		end
+
+		private
+
+		def random_node
+			@fst.m.synchronize { @nodes.sort[@rng.rand(@nodes.length)] }
+		end
+
+		class ExecCommand < Op
+			WEIGHT = 20
+
+			def apply
+				node = random_node
+
+				node.m.synchronize do
+					logger.info(logloc) { "Sending command #{@op_id} to node #{node.t.name}" }
+					print ">"
+
+					begin
+						return if node.crashed?
+						@fst.edn(process: node.t.name, type: :invoke, f: :write, value: @op_id, index: @op_id * 2)
+						node.c.command(@op_id.to_s)
+						@fst.edn(process: node.t.name, type: :ok, f: :write, value: @op_id, index: @op_id * 2 + 1)
+					rescue => ex
+						log_exception(ex) { "Sending command #{@op_id} to node #{node.t.name}" }
+						@fst.edn(process: node.t.name, type: :info, f: :write, value: @op_id, index: @op_id * 2 + 1)
+						node.crashed!
+					end
+
+					logger.info(logloc) { "ExecCommand complete" }
+				end
+#				{ id: @op_id, op: :write, value: @op_id.to_s, node: node.c.node_name, start_time: start.to_f, duration: Time.now - start }
+			end
+		end
+
+		class ReadState < Op
+			WEIGHT = 20
+
+			def apply
+				node = random_node
+
+				node.m.synchronize do
+					logger.info(logloc) { "Reading state from node #{node.t.name}" }
+					print "<"
+
+					begin
+						return if node.crashed?
+						@fst.edn(process: node.t.name, type: :invoke, f: :read, value: nil, index: @op_id * 2)
+						v = node.c.state
+						@fst.edn(process: node.t.name, type: :ok, f: :read, value: v == "" ? nil : v.to_i, index: @op_id * 2 + 1)
+					rescue => ex
+						log_exception(ex) { "Reading state from node #{node.t.name}" }
+						@fst.edn(process: node.t.name, type: :fail, f: :read, value: nil, index: @op_id * 2 + 1)
+						node.crashed!
+					end
+
+					logger.info(logloc) { "ReadState complete" }
+				end
+#				{ id: @op_id, op: :read, value: v, node: node.c.node_name, start_time: start.to_f, duration: Time.now - start }
+			end
+		end
+
+		class CrashNode < Op
+			WEIGHT = 1
+
+			def apply
+				victim = random_node
+
+				@fst.m.synchronize do
+					print "!"
+
+					@nodes.delete(victim)
+
+					begin
+						victim.m.synchronize do
+							node_name, rev = victim.t.name.split("-", 2)
+							rev = rev.to_i + 1
+
+							start = Time.now
+
+							@logger.info(logloc) { "Crashing node #{node_name}" }
+							victim.crashed!
+							victim.t.kill
+
+							@fst.edn(process: victim.t.name, type: :info, f: :crash, index: @op_id * 2)
+
+							if @rng.rand > 0.9
+								@logger.info(logloc) { "Removing existing node state" }
+								$tmpdir.join(node_name, "snapshot.yaml").unlink rescue nil
+								$tmpdir.join(node_name, "log.yaml").unlink rescue nil
+							end
+
+							@logger.info(logloc) { "Spawning replacement #{node_name}" }
+
+							until leader = @nodes.find { |n| n.c.leader? }
+								@logger.info(logloc) { "Waiting until the cluster has a leader" }
+								sleep 0.5
+							end
+
+							lazarus = ClusterNode.new(node_name: node_name, shared_keys: ["s3kr1t"], storage_dir: $tmpdir.join(node_name).to_s, logger: default_logger, join_hints: leader.c.nodes)
+							lazarus.t.name = "#{node_name}-#{rev}"
+
+							@nodes << lazarus
+
+							until lazarus.c.leader? || lazarus.c.follower?
+								@logger.info(logloc) { "Waiting for replacement #{node_name} to join the cluster" }
+								sleep 0.5
+							end
+						rescue => ex
+							log_exception(ex) { "Crashing node #{node_name.inspect}" }
+						end
+					end
+				end
+
+#				{ id: @op_id, op: :crash, node: victim.c.node_name, start_time: start.to_f }
+			end
+		end
+	end
+
+	class Checker
+		def initialize(log)
+			@log = log
+		end
+
+		def valid?
+			valid_values = []
+			final_value = [nil, nil]
+			final_at = nil
+
+			@log.sort_by { |e| e[:start_time] }.each do |entry|
+				if entry[:op] == :write
+				end
+			end
+		end
+	end
+
+	module ClusterNodeAdditions
+		attr_reader :m
+
+		def initialize(*_)
+			@m = Mutex.new
+
+			super
+		end
+	end
+end
+
+ClusterNode.prepend(FinalSmokeTest::ClusterNodeAdditions)
+
+Evinrude.prepend(FinalSmokeTest::EvinrudeHooks)
+
+seed = (ENV["FST_SEED"] || rand(65536)).to_i
+op_count = (ENV["FST_OPS"] || 1000).to_i
+
+puts "Commencing Evinrude Final Smoke Test, seed=#{seed}, op_count=#{op_count}"
+
+st = FinalSmokeTest.new(seed: seed, op_count: op_count, logger: default_logger, interval: 0.05..0.2, edn_file: $tmpdir.join("fst.edn").open("w"))
+
+print "Spawning nodes..."
+nodes = spawn_nodes(5, storage_base_dir: $tmpdir.to_s)
+puts " done."
+
+print "Waiting for stability..."
+wait_for_stability(nodes)
+puts " done."
+
+print "Running ops..."
+st.run(nodes)
+
+#$tmpdir.join("fst.log").write(st.op_log.to_yaml)
+
+#assert FinalSmokeTest::Checker.new(nodes).valid?, "FST oplog validity"
+
+puts " done!"