pontoon 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3d547ad0a54b22d03d8862391c8dfacdffb59fa0
+  data.tar.gz: f0140570f9bc1ad3c756748bf3e2d56ccb8972d7
+SHA512:
+  metadata.gz: 4dfe51bc72db1c90793fd6da0f69bdbca28a2b71360134dbb222bc27b8deed8960b2fde0a71f37222b55154d4f3dbf2260f753cd4ebc27a91005b452f6440ad9
+  data.tar.gz: ed25f5d79a2f4d5526424c3c38fb1a9c59519c42461ac82755f86e8b640714288f633ca5479dfff4c544f9e966041a977daf7ee670acda569734ff563dcccee6

data/lib/pontoon.rb

@@ -0,0 +1,536 @@
+require 'delegate'
+
+module Pontoon
+  Config = Struct.new(:rpc_provider, :async_provider, :election_timeout, :election_splay, :update_interval, :heartbeat_interval)
+
+  class Cluster
+    attr_reader :node_ids
+
+    def initialize(*node_ids)
+      @node_ids = node_ids
+    end
+
+    def quorum
+      @node_ids.count / 2 + 1 # integer division rounds down
+    end
+  end
+
+  class LogEntry
+    attr_reader :term, :index, :command
+
+    def initialize(term, index, command)
+      @term, @index, @command = term, index, command
+    end
+
+    def ==(other)
+      [:term, :index, :command].all? do |attr|
+        self.send(attr) == other.send(attr)
+      end
+    end
+
+    def eql?(other)
+      self == other
+    end
+
+    def hash
+      [:term, :index, :command].reduce(0) do |h, attr|
+        h ^= self.send(attr)
+      end
+    end
+  end
+
+  class Log < DelegateClass(Array)
+    def last(*args)
+      self.any? ? super(*args) : LogEntry.new(nil, nil, nil)
+    end
+  end
+
+  class PersistentState #= Struct.new(:current_term, :voted_for, :log)
+    attr_reader :current_term, :voted_for, :log
+
+    def initialize
+      @current_term = 0
+      @voted_for = nil
+      @log = Log.new([])
+    end
+
+    def current_term=(new_term)
+      raise 'cannot restart an old term' unless @current_term < new_term
+      @current_term = new_term
+      @voted_for = nil
+    end
+
+    def voted_for=(new_votee)
+      raise 'cannot change vote for this term' unless @voted_for.nil?
+      @voted_for = new_votee
+    end
+
+    def log=(new_log)
+      @log = Log.new(new_log)
+    end
+  end
+
+  class TemporaryState
+    attr_reader :commit_index
+    attr_accessor :leader_id
+
+    def initialize(commit_index, leader_id)
+      @commit_index, @leader_id = commit_index, leader_id
+    end
+
+    def commit_index=(new_commit_index)
+      raise 'cannot uncommit log entries' unless @commit_index.nil? || @commit_index <= new_commit_index
+      @commit_index = new_commit_index
+    end
+  end
+
+  class LeadershipState
+    def followers
+      @followers ||= {}
+    end
+
+    attr_reader :update_timer
+
+    def initialize(update_interval)
+      @update_timer = Timer.new(update_interval)
+    end
+  end
+
+  FollowerState = Struct.new(:next_index, :succeeded)
+
+  RequestVoteRequest = Struct.new(:term, :candidate_id, :last_log_index, :last_log_term)
+
+  #class RequestVoteRequest < Struct.new(:term, :candidate_id, :last_log_index, :last_log_term)
+  #  def term; @term.to_i; end
+  #  def last_log_index; @last_log_index.to_i; end
+  #  def last_log_term; @last_log_term.to_i; end
+  #end
+
+  RequestVoteResponse = Struct.new(:term, :vote_granted)
+
+  #class RequestVoteResponse < Struct.new(:term, :vote_granted)
+  #  def term; @term.to_i; end
+  #end
+
+  AppendEntriesRequest = Struct.new(:term, :leader_id, :prev_log_index, :prev_log_term, :entries, :commit_index)
+
+  AppendEntriesResponse = Struct.new(:term, :success)
+
+  CommandRequest = Struct.new(:command)
+
+  CommandResponse = Struct.new(:success)
+
+  class RpcProvider
+    def request_votes(request, cluster)
+      raise "Your RpcProvider subclass must implement #request_votes"
+    end
+
+    def append_entries(request, cluster)
+      raise "Your RpcProvider subclass must implement #append_entries"
+    end
+
+    def append_entries_to_follower(request, node_id)
+      raise "Your RpcProvider subclass must implement #append_entries_to_follower"
+    end
+
+    def command(request, node_id)
+      raise "Your RpcProvider subclass must implement #command"
+    end
+  end
+
+  class AsyncProvider
+    def await
+      raise "Your AsyncProvider subclass must implement #await"
+    end
+  end
+
+  class Timer
+    def initialize(interval, splay=0.0)
+      @interval = interval.to_f
+      @splay = splay.to_f
+      @start = Time.now - @interval + (rand * @splay)
+    end
+
+    def splayed_interval
+      (@interval + (rand * @splay))#.tap {|t|STDOUT.write("\nsplayed interval is #{t}\n")}
+    end
+
+    def reset!
+      @start = Time.now + splayed_interval
+      #STDOUT.write("\ntimer will elapse at #{timeout.strftime('%H:%M:%S:%L')} (timeout is #{timeout.class})\n")
+    end
+
+    def timeout
+      @start + @interval
+    end
+
+    def timed_out?
+      #STDOUT.write("\ntime is #{Time.now.strftime('%M:%S:%L')}\n")
+      Time.now > timeout
+    end
+  end
+
+  class Node
+    attr_reader :id
+    attr_reader :role
+    attr_reader :config
+    attr_reader :cluster
+    attr_reader :persistent_state
+    attr_reader :temporary_state
+    attr_reader :election_timer
+
+    FOLLOWER_ROLE = 0
+    CANDIDATE_ROLE = 1
+    LEADER_ROLE = 2
+
+    def initialize(id, config, cluster, commit_handler=nil, &block)
+      @id = id
+      @role = FOLLOWER_ROLE
+      @config = config
+      @cluster = cluster
+      @persistent_state = PersistentState.new
+      @temporary_state = TemporaryState.new(nil, nil)
+      @election_timer = Timer.new(config.election_timeout, config.election_splay)
+      @commit_handler = commit_handler || (block.to_proc if block_given?)
+    end
+
+    def update
+      return if @updating
+      @updating = true
+      indent = "\t" * (@id.to_i % 3)
+      #STDOUT.write("\n\n#{indent}update #{@id}, role #{@role}, log length #{@persistent_state.log.count}\n\n")
+      case @role
+      when FOLLOWER_ROLE
+        follower_update
+      when CANDIDATE_ROLE
+        candidate_update
+      when LEADER_ROLE
+        leader_update
+      end
+      @updating = false
+    end
+
+    def follower_update
+      if @election_timer.timed_out?
+        #STDOUT.write("follower node #{@id} election timed out at #{Time.now.strftime('%H:%M:%S:%L')}\n")
+        @role = CANDIDATE_ROLE
+        candidate_update
+      end
+    end
+
+    protected :follower_update
+
+    def candidate_update
+      if @election_timer.timed_out?
+        #STDOUT.write("candidate node #{@id} election timed out at #{Time.now.strftime('%H:%M:%S:%L')}\n")
+        @persistent_state.current_term += 1
+        @persistent_state.voted_for = @id
+        reset_election_timeout
+        last_log_entry = @persistent_state.log.last
+        log_index = last_log_entry ? last_log_entry.index : nil
+        log_term = last_log_entry ? last_log_entry.term : nil
+        request = RequestVoteRequest.new(@persistent_state.current_term, @id, log_index, log_term)
+        votes_for = 1 # candidate always votes for self
+        votes_against = 0
+        quorum = @cluster.quorum
+        #STDOUT.write("\n\t\t#{@id} requests votes for term #{@persistent_state.current_term}\n\n")
+        @config.rpc_provider.request_votes(request, @cluster) do |voter_id, request, response|
+          #STDOUT.write("\n\t\t#{@id} receives vote #{response.vote_granted} from #{voter_id}\n\n")
+          elected = nil # no majority result yet
+          if request.term != @persistent_state.current_term
+            # this is a response to an out-of-date request, just ignore it
+          elsif response.term > @persistent_state.current_term
+            @role = FOLLOWER_ROLE
+            elected = false
+          elsif response.vote_granted
+            votes_for += 1
+            elected = true if votes_for >= quorum
+          else
+            votes_against += 1
+            elected = false if votes_against >= quorum
+          end
+          #STDOUT.write("\n\t\t#{@id} receives vote #{response.vote_granted}, elected is #{elected.inspect}\n\n")
+          elected
+        end
+        if votes_for >= quorum
+          #STDOUT.write("\n#{@id} becomes leader for term #{@persistent_state.current_term}\n\n")
+          @role = LEADER_ROLE
+          establish_leadership
+        else
+          #STDOUT.write("\n\t\t#{@id} not elected leader (for #{votes_for}, against #{votes_against})\n\n")
+        end
+      end
+    end
+
+    protected :candidate_update
+
+    def leader_update
+      #STDOUT.write("\nLEADER UPDATE BEGINS\n")
+      if @leadership_state.update_timer.timed_out?
+        @leadership_state.update_timer.reset!
+        send_heartbeats
+      end
+      if @leadership_state.followers.any?
+        new_commit_index = @leadership_state.followers.values.
+            select { |follower_state| follower_state.succeeded }.
+            map { |follower_state| follower_state.next_index - 1 }.
+            sort[@cluster.quorum - 1]
+      else
+        new_commit_index = @persistent_state.log.size - 1
+      end
+      handle_commits(new_commit_index)
+      #STDOUT.write("\nLEADER UPDATE ENDS\n")
+    end
+
+    protected :leader_update
+
+    def handle_commits(new_commit_index)
+      #STDOUT.write("\nnode #{@id} handle_commits(new_commit_index = #{new_commit_index}) (@temporary_state.commit_index = #{@temporary_state.commit_index}\n")
+      return if new_commit_index == @temporary_state.commit_index
+      next_commit = @temporary_state.commit_index.nil? ? 0 : @temporary_state.commit_index + 1
+      while next_commit <= new_commit_index
+        @commit_handler.call(@persistent_state.log[next_commit].command) if @commit_handler
+        @temporary_state.commit_index = next_commit
+        next_commit += 1
+        #STDOUT.write("\n\tnode #{@id} handle_commits(new_commit_index = #{new_commit_index}) (new @temporary_state.commit_index = #{@temporary_state.commit_index}\n")
+      end
+    end
+
+    protected :handle_commits
+
+    def establish_leadership
+      @leadership_state = LeadershipState.new(@config.update_interval)
+      @temporary_state.leader_id = @id
+      @cluster.node_ids.each do |node_id|
+        next if node_id == @id
+        follower_state = (@leadership_state.followers[node_id] ||= FollowerState.new)
+        follower_state.next_index = @persistent_state.log.size
+        follower_state.succeeded = false
+      end
+      send_heartbeats
+    end
+
+    protected :establish_leadership
+
+    def send_heartbeats
+      #STDOUT.write("\nnode #{@id} sending heartbeats at #{Time.now.strftime('%H:%M:%S:%L')}\n")
+      last_log_entry = @persistent_state.log.last
+      log_index = last_log_entry ? last_log_entry.index : nil
+      log_term = last_log_entry ? last_log_entry.term : nil
+      request = AppendEntriesRequest.new(
+          @persistent_state.current_term,
+          @id,
+          log_index,
+          log_term,
+          [],
+          @temporary_state.commit_index)
+
+      @config.rpc_provider.append_entries(request, @cluster) do |node_id, response|
+        append_entries_to_follower(node_id, request, response)
+      end
+    end
+
+    protected :send_heartbeats
+
+    def append_entries_to_follower(node_id, request, response)
+      if @role != LEADER_ROLE
+        # we lost the leadership
+      elsif response.success
+        #STDOUT.write("\nappend_entries_to_follower #{node_id} request #{request.pretty_inspect} succeeded\n")
+        @leadership_state.followers[node_id].next_index = (request.prev_log_index || -1) + request.entries.count + 1
+        @leadership_state.followers[node_id].succeeded = true
+      elsif response.term <= @persistent_state.current_term
+        #STDOUT.write("\nappend_entries_to_follower #{node_id} request failed (#{request.pretty_inspect}) and responded with #{response.pretty_inspect}\n")
+        @config.rpc_provider.append_entries_to_follower(request, node_id) do |node_id, response|
+          if @role == LEADER_ROLE # make sure leadership wasn't lost since the request
+            #STDOUT.write("\nappend_entries_to_follower #{node_id} callback...\n")
+            prev_log_index = (request.prev_log_index.nil? || request.prev_log_index <= 0) ? nil : request.prev_log_index - 1
+            prev_log_term = nil
+            entries = @persistent_state.log
+            unless prev_log_index.nil?
+              prev_log_term = @persistent_state.log[prev_log_index].term
+              entries = @persistent_state.log.slice((prev_log_index + 1)..-1)
+            end
+            next_request = AppendEntriesRequest.new(
+                @persistent_state.current_term,
+                @id,
+                prev_log_index,
+                prev_log_term,
+                entries,
+                @temporary_state.commit_index)
+            #STDOUT.write("\nappend_entries_to_follower #{node_id} request #{request.pretty_inspect} failed...\n")
+            #STDOUT.write("sending updated request #{next_request.pretty_inspect}\n")
+            @config.rpc_provider.append_entries_to_follower(next_request, node_id) do |node_id, response|
+              append_entries_to_follower(node_id, next_request, response)
+            end
+          end
+        end
+      end
+    end
+
+    protected :append_entries_to_follower
+
+    def handle_request_vote(request)
+      #STDOUT.write("\nnode #{@id} handling vote request from #{request.candidate_id} (request.last_log_index: #{request.last_log_index}, vs #{@persistent_state.log.last.index}\n")
+      response = RequestVoteResponse.new
+      response.term = @persistent_state.current_term
+      response.vote_granted = false
+
+      return response if request.term < @persistent_state.current_term
+
+      @temporary_state.leader_id = nil if request.term > @persistent_state.current_term
+
+      step_down_if_new_term(request.term)
+
+      if FOLLOWER_ROLE == @role
+        if @persistent_state.voted_for == request.candidate_id
+          response.vote_granted = true
+        elsif @persistent_state.voted_for.nil?
+          if @persistent_state.log.empty?
+            # this node has no log so it can't be ahead
+            @persistent_state.voted_for = request.candidate_id
+            response.vote_granted = true
+          elsif request.last_log_term == @persistent_state.log.last.term &&
+              (request.last_log_index || -1) < @persistent_state.log.last.index
+            # candidate's log is incomplete compared to this node
+          elsif (request.last_log_term || -1) < @persistent_state.log.last.term
+            # candidate's log is incomplete compared to this node
+          else
+            @persistent_state.voted_for = request.candidate_id
+            response.vote_granted = true
+          end
+        end
+        reset_election_timeout if response.vote_granted
+      end
+
+      response
+    end
+
+    def handle_append_entries(request)
+      #STDOUT.write("\n\nnode #{@id} handle_append_entries: #{request.entries.pretty_inspect}\n\n") #if request.prev_log_index.nil?
+      response = AppendEntriesResponse.new
+      response.term = @persistent_state.current_term
+      response.success = false
+
+      #STDOUT.write("\n\nnode #{@id} handle_append_entries for term #{request.term} (current is #{@persistent_state.current_term})\n")# if request.prev_log_index.nil?
+      return response if request.term < @persistent_state.current_term
+      #STDOUT.write("\n\nnode #{@id} handle_append_entries stage 2\n") if request.prev_log_index.nil?
+
+      step_down_if_new_term(request.term)
+
+      reset_election_timeout
+
+      @temporary_state.leader_id = request.leader_id
+
+      abs_log_index = abs_log_index_for(request.prev_log_index, request.prev_log_term)
+      return response if abs_log_index.nil? && !request.prev_log_index.nil? && !request.prev_log_term.nil?
+      #STDOUT.write("\n\nnode #{@id} handle_append_entries stage 3\n") if request.prev_log_index.nil?
+      if @temporary_state.commit_index &&
+          abs_log_index &&
+          abs_log_index < @temporary_state.commit_index
+        raise "Cannot truncate committed logs; @temporary_state.commit_index = #{@temporary_state.commit_index}; abs_log_index = #{abs_log_index}"
+      end
+
+      truncate_and_update_log(abs_log_index, request.entries)
+
+      return response unless update_commit_index(request.commit_index)
+      #STDOUT.write("\n\nnode #{@id} handle_append_entries stage 4\n") if request.prev_log_index.nil?
+
+      response.success = true
+      response
+    end
+
+    def handle_command(request)
+      response = CommandResponse.new(false)
+      case @role
+      when FOLLOWER_ROLE
+        await_leader
+        if @role == LEADER_ROLE
+          handle_command(request)
+        else
+          # forward the command to the leader
+          response = @config.rpc_provider.command(request, @temporary_state.leader_id)
+        end
+      when CANDIDATE_ROLE
+        await_leader
+        response = handle_command(request)
+      when LEADER_ROLE
+        last_log = @persistent_state.log.last
+        log_entry = LogEntry.new(@persistent_state.current_term, last_log.index ? last_log.index + 1 : 0, request.command)
+        @persistent_state.log << log_entry
+        await_consensus(log_entry)
+        response = CommandResponse.new(true)
+      end
+      response
+    end
+
+    def await_consensus(log_entry)
+      @config.async_provider.await do
+        persisted_log_entry = @persistent_state.log[log_entry.index]
+        !@temporary_state.commit_index.nil? &&
+            @temporary_state.commit_index >= log_entry.index &&
+            persisted_log_entry.term == log_entry.term &&
+            persisted_log_entry.command == log_entry.command
+      end
+    end
+
+    protected :await_consensus
+
+    def await_leader
+      if @temporary_state.leader_id.nil?
+        @role = CANDIDATE_ROLE
+      end
+      @config.async_provider.await do
+        @role != CANDIDATE_ROLE && !@temporary_state.leader_id.nil?
+      end
+    end
+
+    protected :await_leader
+
+    def step_down_if_new_term(request_term)
+      if request_term > @persistent_state.current_term
+        @persistent_state.current_term = request_term
+        @role = FOLLOWER_ROLE
+      end
+    end
+
+    protected :step_down_if_new_term
+
+    def reset_election_timeout
+      @election_timer.reset!
+    end
+
+    protected :reset_election_timeout
+
+    def abs_log_index_for(prev_log_index, prev_log_term)
+      @persistent_state.log.rindex { |log_entry| log_entry.index == prev_log_index && log_entry.term == prev_log_term }
+    end
+
+    protected :abs_log_index_for
+
+    def truncate_and_update_log(abs_log_index, entries)
+      log = @persistent_state.log
+      if abs_log_index.nil?
+        log = []
+      elsif log.length == abs_log_index + 1
+        # no truncation required, past log is the same
+      else
+        log = log.slice(0..abs_log_index)
+      end
+      #STDOUT.write("\n\nentries is: #{entries.pretty_inspect}\n\n")
+      log = log.concat(entries) unless entries.empty?
+      @persistent_state.log = log
+    end
+
+    protected :truncate_and_update_log
+
+    def update_commit_index(new_commit_index)
+      #STDOUT.write("\n\n%%%%%%%%%%%%%%%%%%%%% node #{@id} update_commit_index(new_commit_index = #{new_commit_index})\n")
+      return false if @temporary_state.commit_index && @temporary_state.commit_index > new_commit_index
+      handle_commits(new_commit_index)
+      true
+    end
+
+    protected :update_commit_index
+  end
+end

data/lib/pontoon/goliath.rb

@@ -0,0 +1,248 @@
+require_relative '../pontoon'
+
+require 'goliath'
+
+module Pontoon
+  class Goliath
+
+    def self.log(message)
+      #STDOUT.write("\n\n")
+      #STDOUT.write(message)
+      #STDOUT.write("\n\n")
+    end
+
+    class HttpJsonRpcResponder < ::Goliath::API
+      use ::Goliath::Rack::Render, 'json'
+      use ::Goliath::Rack::Validation::RequestMethod, %w(POST)
+      use ::Goliath::Rack::Params
+
+      def initialize(node)
+        @node = node
+      end
+
+      HEADERS = { 'Content-Type' => 'application/json' }
+
+      def response(env)
+        case env['REQUEST_PATH']
+        when '/request_vote'
+          handle_errors { request_vote_response(env['params']) }
+        when '/append_entries'
+          handle_errors { append_entries_response(env['params']) }
+        when '/command'
+          handle_errors { command_response(env['params']) }
+        else
+          error_response(404, 'not found')
+        end
+      end
+
+      def request_vote_response(params)
+        #STDOUT.write("\nnode #{@node.id} received request_vote from #{params['candidate_id']}, term #{params['term']}\n")
+        request = Pontoon::RequestVoteRequest.new(
+            params['term'],
+            params['candidate_id'],
+            params['last_log_index'],
+            params['last_log_term'])
+        response = @node.handle_request_vote(request)
+        [200, HEADERS, { 'term' => response.term, 'vote_granted' => response.vote_granted }]
+      end
+
+      def append_entries_response(params)
+        #STDOUT.write("\nnode #{@node.id} received append_entries from #{params['leader_id']}, term #{params['term']}\n")
+        entries = params['entries'].map {|entry| Pontoon::LogEntry.new(entry['term'], entry['index'], entry['command'])}
+        request = Pontoon::AppendEntriesRequest.new(
+            params['term'],
+            params['leader_id'],
+            params['prev_log_index'],
+            params['prev_log_term'],
+            entries,
+            params['commit_index'])
+        #STDOUT.write("\nnode #{@node.id} received entries: #{request.entries.pretty_inspect}\n")
+        response = @node.handle_append_entries(request)
+        #STDOUT.write("\nnode #{@node.id} completed append_entries from #{params['leader_id']}, term #{params['term']} (#{response})\n")
+        [200, HEADERS, { 'term' => response.term, 'success' => response.success }]
+      end
+
+      def command_response(params)
+        request = Pontoon::CommandRequest.new(params['command'])
+        response = @node.handle_command(request)
+        [response.success ? 200 : 409, HEADERS, { 'success' => response.success }]
+      end
+
+      def handle_errors
+        yield
+      rescue StandardError => se
+        error_response(422, se)
+      rescue Exception => e
+        error_response(500, e)
+      end
+
+      def error_message(exception)
+        "#{exception.message}\n\t#{exception.backtrace.join("\n\t")}".tap {|m| STDOUT.write("\n\n\t#{m}\n\n")}
+      end
+
+      def error_response(code, exception)
+        [code, HEADERS, { 'error' => error_message(exception) }]
+      end
+    end
+
+    module HashMarshalling
+      def self.hash_to_object(hash, klass)
+        object = klass.new
+        hash.each_pair do |k, v|
+          object.send("#{k}=", v)
+        end
+        object
+      end
+
+      def self.object_to_hash(object, attrs)
+        attrs.reduce({}) { |hash, attr|
+          hash[attr] = object.send(attr); hash
+        }
+      end
+    end
+
+    class HttpJsonRpcProvider < Pontoon::RpcProvider
+      attr_reader :uri_generator
+
+      def initialize(uri_generator)
+        @uri_generator = uri_generator
+      end
+
+      def request_votes(request, cluster, &block)
+        sent_hash = HashMarshalling.object_to_hash(request, %w(term candidate_id last_log_index last_log_term))
+        sent_json = MultiJson.dump(sent_hash)
+        deferred_calls = []
+        EM.synchrony do
+          cluster.node_ids.each do |node_id|
+            next if node_id == request.candidate_id
+            http = EventMachine::HttpRequest.new(uri_generator.call(node_id, 'request_vote')).apost(
+                :body => sent_json,
+                :head => { 'Content-Type' => 'application/json' })
+            http.callback do
+              if http.response_header.status == 200
+                received_hash = MultiJson.load(http.response)
+                response = HashMarshalling.hash_to_object(received_hash, Pontoon::RequestVoteResponse)
+                #STDOUT.write("\n\t#{node_id} responded #{response.vote_granted} to #{request.candidate_id}\n\n")
+                yield node_id, request, response
+              else
+                Pontoon::Goliath.log("request_vote failed for node '#{node_id}' with code #{http.response_header.status}")
+              end
+            end
+            deferred_calls << http
+          end
+        end
+        deferred_calls.each do |http|
+          EM::Synchrony.sync http
+        end
+      end
+
+      def append_entries(request, cluster, &block)
+        deferred_calls = []
+        EM.synchrony do
+          cluster.node_ids.each do |node_id|
+            next if node_id == request.leader_id
+            deferred_calls << create_append_entries_to_follower_request(request, node_id, &block)
+          end
+        end
+        deferred_calls.each do |http|
+          EM::Synchrony.sync http
+        end
+      end
+
+      def append_entries_to_follower(request, node_id, &block)
+#        EM.synchrony do
+          create_append_entries_to_follower_request(request, node_id, &block)
+#        end
+      end
+
+      def create_append_entries_to_follower_request(request, node_id, &block)
+        sent_hash = HashMarshalling.object_to_hash(request, %w(term leader_id prev_log_index prev_log_term entries commit_index))
+        sent_hash['entries'] = sent_hash['entries'].map {|obj| HashMarshalling.object_to_hash(obj, %w(term index command))}
+        sent_json = MultiJson.dump(sent_hash)
+        raise "replicating to self!" if request.leader_id == node_id
+        #STDOUT.write("\nleader #{request.leader_id} replicating entries to #{node_id}: #{sent_hash.pretty_inspect}\n")#"\t#{caller[0..4].join("\n\t")}")
+
+        http = EventMachine::HttpRequest.new(uri_generator.call(node_id, 'append_entries')).apost(
+            :body => sent_json,
+            :head => { 'Content-Type' => 'application/json' })
+        http.callback do
+          #STDOUT.write("\nleader #{request.leader_id} calling back to #{node_id} to append entries\n")
+          if http.response_header.status == 200
+            received_hash = MultiJson.load(http.response)
+            response = HashMarshalling.hash_to_object(received_hash, Pontoon::AppendEntriesResponse)
+            yield node_id, response
+          else
+            Pontoon::Goliath.log("append_entries failed for node '#{node_id}' with code #{http.response_header.status}")
+          end
+        end
+        http
+      end
+
+      def command(request, node_id)
+        sent_hash = HashMarshalling.object_to_hash(request, %w(command))
+        sent_json = MultiJson.dump(sent_hash)
+        http = EventMachine::HttpRequest.new(uri_generator.call(node_id, 'command')).apost(
+            :body => sent_json,
+            :head => { 'Content-Type' => 'application/json' })
+        http = EM::Synchrony.sync(http)
+        if http.response_header.status == 200
+          received_hash = MultiJson.load(http.response)
+          HashMarshalling.hash_to_object(received_hash, Pontoon::CommandResponse)
+        else
+          Pontoon::Goliath.log("command failed for node '#{node_id}' with code #{http.response_header.status}")
+          CommandResponse.new(false)
+        end
+      end
+    end
+
+    class EventMachineAsyncProvider < Pontoon::AsyncProvider
+      def await
+        f = Fiber.current
+        until yield
+          EM.next_tick {f.resume}
+          Fiber.yield
+        end
+      end
+    end
+
+    def self.rpc_provider(uri_generator)
+      HttpJsonRpcProvider.new(uri_generator)
+    end
+
+    def self.async_provider
+      EventMachineAsyncProvider.new
+    end
+
+    def initialize(node)
+      @node = node
+    end
+
+    attr_reader :node
+    attr_reader :update_fiber
+    attr_reader :running
+
+    def start(options = {})
+      @runner = ::Goliath::Runner.new(ARGV, nil)
+      @runner.api = HttpJsonRpcResponder.new(node)
+      @runner.app = ::Goliath::Rack::Builder.build(HttpJsonRpcResponder, @runner.api)
+      @runner.address = options[:address] if options[:address]
+      @runner.port = options[:port] if options[:port]
+      @runner.run
+      @running = true
+
+      update_proc = Proc.new do
+        EM.synchrony do
+          @node.update
+        end
+      end
+      @update_timer = EventMachine.add_periodic_timer(node.config.update_interval, update_proc)
+#      @node.update
+    end
+
+    def stop
+      @update_timer.cancel
+    end
+  end
+end
+
+

metadata

@@ -0,0 +1,115 @@
+--- !ruby/object:Gem::Specification
+name: pontoon
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Anthony Corletti
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-04-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: goliath
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: cucumber
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: em-http-request
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+description: A simple Raft distributed consensus implementation
+email: anthcor@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/pontoon.rb
+- lib/pontoon/goliath.rb
+homepage: http://github.com/anthcor/pontoon
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: A simple Raft distributed consensus implementation
+test_files: []