zeevex_cluster 0.2.1

data/lib/zeevex_cluster/serializer/json_hash.rb

@@ -0,0 +1,67 @@
+require 'json'
+require 'json/add/core'
+require 'date'
+
+require 'zeevex_cluster/serializer'
+
+class ZeevexCluster::Serializer::JsonHash
+  def new(options = {})
+    @options = options
+  end
+
+  def is_time_field(key, val = nil)
+    key.to_s.match(/(_at|timestamp)$/)
+  end
+
+  def untranslate_hash(parsed)
+    raise ArgumentError, 'Must be a hash' unless parsed.is_a?(Hash)
+    if parsed.count == 1 && (parsed.has_key?('$primitive') || parsed.has_key?(:$primitive))
+      return parsed.values.first
+    end
+    hash = {}
+    parsed.each do |(key, val)|
+      # val = Time.at(val).utc if is_time_field(key, val)
+      hash[key.to_sym] = val
+    end
+    hash
+  end
+
+  def translate_hash(hash)
+    raise ArgumentError, 'Must be a hash' unless hash.is_a?(Hash)
+    hash = hash.clone
+    #hash.keys.each do |key|
+    #  hash[key] = hash[key].utc.to_f if is_time_field(key, hash[key])
+    #end
+    hash
+  end
+
+  def deserialize(str)
+    parsed = JSON.parse(str, :symbolize_names => true, 
+                             :object_class => IndifferentHash,
+                             :create_additions => true)
+    case parsed
+      when Hash then untranslate_hash(parsed)
+      else parsed
+    end
+  end
+
+  def serialize(obj)
+    obj = case obj
+            when Hash then translate_hash(obj)
+            when Numeric, String, TrueClass, FalseClass, NilClass then
+                  {'$primitive' => obj}
+            else obj
+          end
+    JSON.dump(obj)
+  end
+
+  class IndifferentHash < Hash
+    def fetch(key, defaultval = nil)
+      super(key.to_sym, defaultval)
+    end
+
+    def [](key)
+      super(key.to_sym)
+    end
+  end
+end

data/lib/zeevex_cluster/serializer.rb

@@ -0,0 +1,27 @@
+
+module ZeevexCluster
+  module Serializer
+    def included(base)
+      base.extend(ClassMethods)
+      base.class_eval { include ZeevexCluster::Serializer::InstanceMethods }
+    end
+
+    module InstanceMethods
+      def to_json
+        serializer.serialize(self)
+      end
+
+      def serializer
+        @_serializer ||= ZeevexCluster::Serializer::JsonHash.new
+      end
+    end
+
+    module ClassMethods
+      def from_json(string)
+        serializer.deserialize(self)
+      end
+    end
+  end
+end
+
+require 'zeevex_cluster/serializer/json_hash'

data/lib/zeevex_cluster/static.rb

@@ -0,0 +1,67 @@
+module ZeevexCluster
+  class Static < Base
+    def initialize(options = {})
+      super
+      raise ArgumentError, "Must supply :master_nodename" unless @options[:master_nodename]
+      if @options[:master_nodename] == :self
+        @options[:master_nodename] = nodename
+      end
+      @member = false
+      after_initialize
+    end
+
+    ##
+    ## joining is a no-op for ol' singleton here
+    ##
+    def join
+      @member = true
+    end
+
+    ##
+    ## leaving, too
+    ##
+    def leave
+      @member = false
+      true
+    end
+
+    def member?
+      !! @member
+    end
+
+    ##
+    ## Are we the chosen one?
+    ##
+    def master?
+      member? && nodename == options[:master_nodename]
+    end
+
+    ##
+    ## Nobody can change the master
+    ##
+    def make_master!
+      raise ClusterActionFailed, "Can not change master" unless master?
+      true
+    end
+
+    ##
+    ## Resign from mastership; returns false if this is the only node.
+    ##
+    ## No-op for now.
+    ##
+    def resign!
+      raise NotMaster unless master?
+
+      # master is currently fixed, so we can't resign
+      raise ClusterPolicyViolation, "Current master cannot resign in this implementation."
+    end
+
+    ##
+    ## Return name of master node
+    ##
+    def master
+      options[:master_nodename]
+    end
+
+  end
+end

data/lib/zeevex_cluster/strategy/base.rb

@@ -0,0 +1,92 @@
+require 'zeevex_cluster/strategy'
+require 'hookem'
+
+# require 'zeevex_threadsafe/thread_safer'
+
+module ZeevexCluster::Strategy
+  class Base
+    include ZeevexCluster::Util::Logging
+    include Hookem
+    # include ZeevexThreadsafe::ThreadSafer
+
+    def initialize(options = {})
+      @options       = options
+      @namespace     = options[:namespace]
+      @cluster_name  = options[:cluster_name]
+      @nodename      = options[:nodename] || Socket.gethostname
+      @hooks         = {}
+      @logger        = options[:logger]
+
+      @state         = :stopped
+
+      reset_state_vars
+
+      _initialize_hook_module
+
+      if options[:hooks]
+        add_hooks options[:hooks]
+      end
+    end
+
+    def has_master?
+      !! @current_master
+    end
+
+    def am_i_master?
+      @my_cluster_status == :master
+    end
+
+    def state
+      @state
+    end
+
+    def online?
+      @cluster_status == :online
+    end
+
+    def member?
+      online?
+    end
+
+    def started?
+      @state == :started
+    end
+
+    def stopped?
+      @state == :stopped
+    end
+
+    protected
+
+    def change_my_status(status, attrs = {})
+      return if status == @my_cluster_status
+
+      old_status = @my_cluster_status
+      @my_cluster_status = status
+      run_hook :status_change, status, old_status, attrs
+    end
+
+    def change_master_status(status, attrs = {})
+      return if status == @master_status
+
+      old_status, @master_status = @master_status, status
+      run_hook :master_status_change, status, old_status, attrs
+    end
+
+    def change_cluster_status(status, attrs = {})
+      return if status == @cluster_status
+
+      old_status, @cluster_status = @cluster_status, status
+      run_hook :cluster_status_change, status, old_status, attrs
+    end
+
+    def reset_state_vars
+      @state = :stopped
+      @my_cluster_status = :nonmember
+      @master_status = :none
+      @cluster_status = :offline
+    end
+
+    # make_thread_safe :change_my_status, :change_master_status, :change_cluster_status
+  end
+end

data/lib/zeevex_cluster/strategy/cas.rb

@@ -0,0 +1,403 @@
+require 'zeevex_cluster/strategy/base'
+require 'socket'
+require 'logger'
+
+class ZeevexCluster::Strategy::Cas < ZeevexCluster::Strategy::Base
+
+  attr_accessor :stale_time, :update_period, :server, :nodename, :cluster_name
+
+  SUSPECT_MISSED_UPDATE_COUNT = 3
+  INAUGURATION_UPDATE_DELAY   = 2
+
+  def initialize(options = {})
+    super
+    @stale_time    = options.fetch(:stale_time, 40)
+    @update_period = options.fetch(:update_period, 10)
+
+    unless (@server = options[:coordinator])
+      coordinator_type = options[:coordinator_type] || 'memcached'
+      @server = ZeevexCluster::Coordinator.create(coordinator_type,
+                                                  {:server     => options[:server],
+                                                   :port       => options[:port],
+                                                   :expiration => @stale_time * 4}.merge(options[:coordinator_options] || {}))
+    end
+    unless @server.is_a?(ZeevexCluster::Synchronized)
+      @server = ZeevexCluster.Synchronized(@server)
+    end
+  end
+
+  def do_i_hold_lock?
+    @my_cluster_status == :master || @my_cluster_status == :master_elect
+  end
+
+  def master_node
+    @current_master
+  end
+
+  def master_nodename
+    @current_master && @current_master[:nodename]
+  end
+
+
+
+  class StopException < StandardError; end
+
+  def start
+    raise "Already started" if @thread || @state == :started
+    @start_time = time_now
+    @state  = :started
+    @locked_at = nil
+    @thread = Thread.new do
+      begin
+        change_my_status :member
+        spin
+      rescue
+        logger.warn "rescued from spin: #{$!.inspect}\n#{$!.backtrace.join("\n")}"
+      ensure
+        logger.debug "spin over"
+        @state = :stopped
+      end
+    end
+  end
+
+  def stop
+    case @state
+      when :stop_requested
+      when :stopped
+      when :started
+        @state = :stop_requested
+        @thread.raise(StopException.new 'stop')
+      else
+        raise "Bad state: #{@state}"
+    end
+    @thread.join
+    @thread = nil
+    change_my_status :nonmember
+    reset_state_vars
+  end
+
+  def resign(delay = nil)
+    # unresign
+    if delay == 0
+      @resign_until = nil
+      campaign
+    else
+      @resign_until = time_now + (delay || [@update_period*6, @stale_time].min)
+      current = nil
+      server.cas(key) do |val|
+        current = val
+        if is_me?(val)
+          my_token.merge(:timestamp => time_now - 2*@stale_time)
+        else
+          raise ZeevexCluster::Coordinator::DontChange
+        end
+      end
+      failed_lock(my_token, current)
+    end
+  rescue ZeevexCluster::Coordinator::ConnectionError
+    failed_lock(my_token, nil)
+  end
+
+
+  def steal_election!
+    logger.warn "Stealing election"
+    @resign_until = nil
+    me = my_token
+    server.set(key, me)
+    got_lock(me)
+    true
+  rescue ZeevexCluster::Coordinator::ConnectionError
+    false
+  end
+
+  def members
+    stale_point = time_now - @stale_time
+    list = server.get(key('members')) || make_member_list
+    members = []
+    list[:members].values.each do |v|
+      members << v[:nodename] unless v[:timestamp].utc < stale_point
+    end
+    members
+  end
+
+  protected
+
+  def spin
+    logger.debug "spin started"
+    @state = :started
+    run_hook :started
+    run_hook :joined_cluster, cluster_name
+    while @state == :started
+      begin
+        register
+        campaign
+        if @state == :started
+          begin
+            sleep [@update_period - 1, 1].max
+          rescue StopException
+            logger.debug 'Stopping on stop exception'
+          end
+        end
+      rescue ZeevexCluster::Coordinator::ConnectionError
+        connection_error
+      end
+    end
+  ensure
+    ignoring_connection_error { resign } if do_i_hold_lock?
+    ignoring_connection_error { unregister }
+    @state = :stopped
+    run_hook :left_cluster, cluster_name
+    change_cluster_status :offline
+    run_hook :stopped
+  end
+
+  def ignoring_connection_error
+    begin
+      yield
+    rescue ZeevexCluster::Coordinator::ConnectionError
+      logger.debug 'got connection error in ignoring_connection_error'
+      $!
+    end
+  end
+
+  def connection_error
+    run_hook :connection_error
+    change_cluster_status :offline
+  end
+  
+  def my_token
+    now = time_now
+    {:nodename    => nodename,
+     :joined_at   => @start_time,
+     :locked_at   => @locked_at || now,
+     :timestamp   => now}
+  end
+
+  def key(subkey = 'throne')
+    (@options[:cluster_key] || cluster_name) + ":" + subkey
+  end
+
+  def is_me?(token)
+    token && token.is_a?(Hash) && token[:nodename] == nodename
+  end
+
+
+  def got_lock(token)
+    unless @locked_at
+      @locked_at     = token[:timestamp]
+      token          = my_token
+      run_hook :election_won
+    end
+    @my_master_token = token
+    if qualifies_for_master?(token)
+      change_my_status :master
+      if @current_master && is_me?(@current_master)
+        run_hook :reelected
+      else
+        run_hook :became_master
+      end
+      change_master_status :good
+      @current_master  = token
+    else
+      change_my_status :master_elect
+      change_master_status :waiting_for_inauguration
+      run_hook :waiting_for_inauguration
+      @current_master  = nil
+    end
+  end
+
+  def failed_lock(me, winner)
+    @locked_at       = nil
+
+    if qualifies_for_master?(winner)
+      @current_master = winner
+      change_my_status :member
+      change_master_status :good
+    elsif ! token_invalid?(winner)
+      @current_master = winner
+      change_master_status :waiting_for_inauguration
+    else
+      @current_master = nil
+      change_master_status :none
+    end
+    run_hook :election_lost, @current_master
+
+    if @my_cluster_status == :master
+      @my_master_token = nil
+      change_my_status :lame_duck
+      run_hook :lame_duck
+    else
+      change_my_status :member
+    end
+  end
+
+  #
+  # Must have held lock for INAUGURATION_UPDATE_DELAY update periods
+  #
+  def qualifies_for_master?(token)
+    now = time_now()
+    ! token_invalid?(token) and
+        token[:timestamp] > (now - @stale_time) and
+        token[:locked_at] <= (now - INAUGURATION_UPDATE_DELAY * @update_period)
+  end
+
+  def time_now
+    Time.now.utc
+  end
+
+  def token_invalid?(token)
+    now = time_now
+    !token || !token.is_a?(Hash) || !token[:timestamp] ||
+        ! token[:locked_at] || ! token[:nodename] ||
+        token[:timestamp].utc < (now - @stale_time)
+  end
+
+  def resigned?
+    @resign_until && @resign_until > time_now
+  end
+
+  def campaign
+    me = my_token
+
+    act_resigned = resigned?
+    compete_for_token = !act_resigned
+
+    hook = nil
+    current = nil
+    res = server.cas(key) do |val|
+      current = val
+      if is_me?(val) && !token_invalid?(val) && compete_for_token
+        me
+      elsif token_invalid?(val) && compete_for_token
+        if is_me?(val)
+          logger.info "My old token is invalid, refreshing: #{val.inspect}"
+        else
+          logger.info "CAS: master invalid, stealing: #{val.inspect}"
+          # it's necessary to run this outside of the CAS block to be sure we won
+          hook = :deposed_master
+        end
+        me
+      else
+        run_hook :suspect_master if @master_status != :none && master_suspect?(val)
+        raise ZeevexCluster::Coordinator::DontChange
+      end
+    end
+
+    # if we got a result, we must be online
+    change_cluster_status :online
+
+    if act_resigned
+      run_hook :staying_resigned
+      failed_lock(me, current)
+      return
+    else
+      @resign_until = nil
+    end
+
+    if res
+      run_hook hook if hook && res
+      got_lock(me)
+      return true
+    elsif res.nil?
+      failed_lock(me, nil)
+      if server.add(key, me)
+        logger.debug 'CAS: added frist post!'
+        got_lock(me)
+        return true
+      end
+    end
+
+    # CAS succeeded so we're the boss
+    if res
+      got_lock(me)
+      true
+
+    # didn't get it, somebody else must be boss
+    else
+      failed_lock(me, current)
+      false
+    end
+  rescue ZeevexCluster::Coordinator::ConnectionError
+    connection_error
+    failed_lock(me, current)
+    false
+  end
+
+  def make_member_list
+    {:members => {@nodename => my_token}}
+  end
+
+  def register
+    me = my_token
+
+    self_key = self.key('member:' +  @nodename)
+    memberlist_key = self.key('members')
+    server.set(self_key, me) or raise "failed to set #{self_key}"
+
+    res = false
+    retries = 5
+
+    while retries > 0 && res == false
+      stale_point = time_now - @stale_time
+      res = server.cas(memberlist_key) do |hash|
+        hash[:members] ||= {}
+        hash[:members].keys.each do |key|
+          hash[:members].delete(key) if hash[:members][key][:timestamp] < stale_point
+        end
+        hash[:members][@nodename] = me
+        hash
+      end
+      retries -= 1
+    end
+
+    if res.nil?
+      server.add(memberlist_key, {:members => {@nodename => me}})
+    end
+
+    true
+  rescue ZeevexCluster::Coordinator::ConnectionError
+    connection_error
+    false
+  end
+
+  def unregister
+    me = my_token
+
+    self_key = self.key('member:' +  @nodename)
+    memberlist_key = self.key('members')
+    server.delete(self_key)
+
+    res = false
+    retries = 5
+
+    while retries > 0 && res == false
+      res = server.cas(memberlist_key) do |hash|
+        hash[:members] ||= {}
+        hash[:members].delete @nodename
+        hash
+      end
+      retries -= 1
+    end
+
+    true
+  rescue ZeevexCluster::Coordinator::ConnectionError
+    connection_error
+    false
+  end
+
+  #
+  # has the master gone without updating suspiciously long?
+  #
+  def master_suspect?(token)
+    time_now - token[:timestamp] > SUSPECT_MISSED_UPDATE_COUNT * @update_period
+  end
+
+  def reset_state_vars
+    super
+
+    @resign_until = nil
+    @my_master_token = nil
+    @current_master = nil
+    @thread = nil
+  end
+end

data/lib/zeevex_cluster/strategy/static.rb

@@ -0,0 +1,55 @@
+require 'zeevex_cluster/strategy/base'
+
+module ZeevexCluster::Strategy
+  class Static < Base
+    def initialize(options = {})
+      super
+      @master_nodename = options[:master_nodename] || raise(ArgumentError, 'Must specify :master_nodename')
+      @members = options[:members]
+    end
+
+    def start
+      @state  = :started
+      change_cluster_status :online
+      if @nodename == @master_nodename
+        change_my_status :master
+        change_master_status :good
+      else
+        change_my_status :member
+        change_master_status :unknown
+      end
+    end
+
+    def stop
+      @state = :stopped
+      change_my_status :nonmember
+      change_master_status :unknown
+      change_cluster_status :offline
+    end
+
+    def am_i_master?
+      @state == :started && @my_cluster_status == :master
+    end
+
+    # FIXME: this is CAS-specific
+    def master_node
+      {:nodename => @master_nodename}
+    end
+
+    def members
+      @members || [@master_nodename, @nodename].select {|x| x != "none" }.uniq
+    end
+
+    def resign(delay = nil)
+      # master is currently fixed, so we can't resign
+      logger.warn 'Current master cannot resign in this implementation.'
+      false
+    end
+
+    def steal_election!
+      raise ClusterActionFailed, 'Can not change master' unless am_i_master?
+      true
+    end
+
+  end
+end

data/lib/zeevex_cluster/strategy/unclustered.rb

@@ -0,0 +1,9 @@
+module ZeevexCluster::Strategy
+  class Unclustered < Static
+    def initialize(options)
+      options[:master_nodename] = options[:nodename]
+      super
+      @members = [@nodename]
+    end
+  end
+end