dcell 0.9.0 → 0.10.0

data/lib/dcell/node_manager.rb

@@ -0,0 +1,81 @@
+module DCell
+  # Manage nodes we're connected to
+  class NodeManager
+    include Celluloid::ZMQ
+    include Enumerable
+
+    attr_reader :gossip_rate, :heartbeat_timeout
+
+    def initialize
+      @nodes = {}
+
+      @gossip_rate       = 5  # How often to send gossip in seconds
+      @heartbeat_timeout = 10 # How soon until a lost heartbeat triggers a node partition
+      each { |node| node.socket if node } # Connect all so we can gossip
+      @gossip = after(gossip_rate) { gossip_timeout }
+    end
+
+    # Return all available nodes in the cluster
+    def all
+      Directory.all.map do |node_id|
+        find node_id
+      end
+    end
+
+    # Iterate across all available nodes
+    def each
+      Directory.all.each do |node_id|
+        yield find node_id
+      end
+    end
+
+    # Find a node by its node ID
+    def find(id)
+      node = @nodes[id]
+      return node if node
+
+      addr = Directory[id]
+      return unless addr
+
+      if id == DCell.id
+        node = DCell.me
+      else
+        node = Node.new(id, addr)
+      end
+
+      @nodes[id] ||= node
+      @nodes[id]
+    end
+    alias_method :[], :find
+
+    # Send gossip to a random node (except ourself) after the given interval
+    def gossip_timeout
+      nodes = select { |node| node.state == :connected }
+      peer = nodes.select { |node| node.id != DCell.id }.sample(1)[0]
+      if peer
+        nodes = nodes.inject([]) { |a,n| a << [n.id, n.addr, n.timestamp]; a }
+        data = nil
+        if DCell.registry.is_a? Registry::GossipAdapter
+          data = peer.fresh? ? DCell.registry.values : DCell.registry.changed
+        end
+        DCell.me.tick
+        peer.send_message DCell::Message::Gossip.new nodes, data
+      end
+      @gossip = after(gossip_rate) { gossip_timeout }
+    end
+
+    def handle_gossip(peers, data)
+      peers.each do |id, addr, timestamp|
+        if (node = find(id))
+          node.handle_timestamp! timestamp
+        else
+          Directory[id] = addr
+          Celluloid::Logger.info "Found node #{id}"
+        end
+      end
+      if DCell.registry.is_a? Registry::GossipAdapter
+        data.map { |data| DCell.registry.observe data } if data
+      end
+    end
+  end
+end

data/lib/dcell/registries/cassandra_adapter.rb

@@ -0,0 +1,86 @@
+require 'cassandra'
+
+# create the keyspace / columnfamily with cqlsh
+#
+# create keyspace dcell
+#  with strategy_class='SimpleStrategy'
+#   and strategy_options:replication_factor=3;
+#
+# create columnfamily dcell (dcell_type ascii primary key);
+
+# not sure this is right yet ...
+# Keyspace "whatever" [
+#   ColumnFamily "dcell" {
+#     RowKey "nodes": {
+#       <nodeid>: <address>,
+#       <nodeid>: <address>,
+#       ...
+#     }
+#     RowKey "globals": {
+#       <key>: <marshal blob>,
+#       <key>: <marshal blob>,
+#       ...
+#     }
+#   }
+# ]
+#
+
+module DCell
+  module Registry
+    class CassandraAdapter
+      DEFAULT_KEYSPACE = "dcell"
+      DEFAULT_CF = "dcell"
+
+      def initialize(options)
+        # Convert all options to symbols :/
+        options = options.inject({}) { |h,(k,v)| h[k.to_sym] = v; h }
+
+        keyspace = options[:keyspace] || DEFAULT_KEYSPACE
+        columnfamily = options[:columnfamily] || DEFAULT_CF
+
+        options[:servers] ||= []
+        options[:servers] << options[:server] if options[:server]
+        options[:servers] << "localhost:9160" unless options[:servers].any?
+
+        cass = Cassandra.new(keyspace, options[:servers])
+
+        @global_registry = GlobalRegistry.new(cass, columnfamily)
+      end
+
+      def clear_globals
+        @global_registry.clear
+      end
+
+      class GlobalRegistry
+        def initialize(cass, cf)
+          @cass = cass
+          @cf = cf
+        end
+
+        def get(key)
+          string = @cass.get @cf, "globals", key.to_s
+          Marshal.load string if string
+        end
+
+        # Set a global value
+        def set(key, value)
+          string = Marshal.dump value
+          @cass.insert @cf, "globals", { key.to_s => string }
+        end
+
+        # The keys to all globals in the system
+        def global_keys
+          @cass.get(@cf, "globals").keys
+        end
+
+        def clear
+          @cass.del @cf, "globals"
+        end
+      end
+
+      def get_global(key);        @global_registry.get(key) end
+      def set_global(key, value); @global_registry.set(key, value) end
+      def global_keys;            @global_registry.global_keys end
+    end
+  end
+end

data/lib/dcell/registries/gossip/core.rb

@@ -0,0 +1,235 @@
+require 'forwardable'
+require 'uri'
+
+module DCell
+  module Gossip
+    class VersionVector
+      extend Forwardable
+
+      class Status
+        STATES = [:precedes, :equal, :concurrent, :succeeds]
+        
+        STATES.each do |state|
+          class_eval %Q{
+            def #{state}?
+              @state == '#{state}'.to_sym
+            end
+            def #{state}!
+              @state = '#{state}'.to_sym
+            end
+          }
+        end
+      end
+
+      def_delegators :@versions, :[]
+      attr_reader :versions
+      
+      def initialize(id)
+        @versions = {}
+        update_at id
+      end
+
+      def update_at(id)
+        observe id
+        @versions[id] += 1
+      end
+      
+      def observe(id)
+        @versions[id] ||= 0
+      end
+
+      def covers?(nodes, vector)
+        nodes.each do |node|
+          # We must have an entry for the node
+          return false unless @versions.keys.include? node
+          
+          # And someone else must have seen the entry, too.
+          version = vector.versions[node] || 0
+          return false if @versions[node] != version
+        end
+        true
+      end
+
+      def compare(other)
+        v1_bigger = false
+        v2_bigger = false
+
+        @versions.each do |id, version|
+          if not other.versions.include? id
+            # Version vectors behave like vector clocks, with slightly
+            # different update rules. A vector clock assumes that all
+            # processes initially observe version 0. Since we don't
+            # know the topology ahead of time, we assume that a missing
+            # entry corresponds to a node that has not yet been discovered,
+            # and thus the version is implicitly 0.
+            v1_bigger = true if version > 0
+          else
+            v1_bigger = true if version > other.versions[id]
+            v2_bigger = true if version < other.versions[id]
+          end
+        end
+
+        other.versions.each do |id, version|
+          if not @versions.include? id
+            # See the comment above for the similar v1_bigger calculation.
+            v2_bigger = true if version > 0
+          else
+            v2_bigger = true if version > @versions[id]
+            v1_bigger = true if version < @versions[id]
+          end
+        end
+
+        status = Status.new
+        if !v1_bigger
+          if !v2_bigger
+            status.equal!
+          else
+            status.precedes!
+          end
+        elsif !v2_bigger
+          status.succeeds!
+        else
+          status.concurrent!
+        end
+        return status
+      end
+
+      # Take the entrywise maximum of the versions
+      def merge!(other)
+        @versions.each do |id, version|
+          if other.versions.include? id
+            @versions[id] = other.versions[id] if other.versions[id] > version
+          end
+        end
+
+        other.versions.each do |id, version|
+          if @versions.include? id
+            @versions[id] = @versions[id] if @versions[id] > version
+          end
+        end
+        @versions.merge!(other.versions.reject { |k,v| @versions.include? k })
+      end
+
+      def to_s
+        @versions.to_s
+      end
+    end
+
+    class Store
+      class Data
+        attr_reader :key, :value, :vector
+        def initialize(key, value, id)
+          @key     = key
+          @value   = value
+          @vector  = VersionVector.new(id)
+          @changed = true
+        end
+
+        def clear
+          @vector.update_at DCell.id
+          @deleted = true
+          @value   = nil
+          @changed = true
+        end
+
+        def deleted?
+          @deleted
+        end
+
+        def changed?
+          @changed
+        end
+
+        def observe
+          @vector.observe DCell.id
+        end
+
+        def value=(value)
+          if @value != value
+            @vector.update_at DCell.id
+            @value   = value
+            @deleted = value.nil?
+            @changed = true
+            Celluloid::Logger.debug "Updated key #{key} to #{value}"
+          end
+        end
+
+        def merge!(other)
+          # We'll take other if we preceded it, or if we are
+          # concurrent with it (though issue a warning that data
+          # has been lost).
+          status = @vector.compare(other.vector)
+          if status.precedes? or status.concurrent?
+            if other.value != @value
+              if status.concurrent?
+                Celluloid::Logger.debug "Dropping local copy of concurrent data for #{@key}"
+              else
+                Celluloid::Logger.debug "Observed updated data #{key} => #{other.value}"
+              end
+              @value   = other.value
+              @deleted = @value.nil?
+            end
+          elsif status.succeeds?
+            Celluloid::Logger.debug "Local data succeeds for #{@key}"
+          end
+          @vector.merge!(other.vector) unless status.equal?
+
+          # Stop gossiping if this has been seen by every known, healthy node
+          nodes = DCell::Node.all.map { |node| node.state == :connected }
+          @changed = false if @vector.covers?(nodes, other.vector)
+        end
+      end
+
+      def initialize(base_path)
+        @base_path = base_path
+        @data = {}
+      end
+
+      def path_for(key)
+        "#{@base_path}/#{key}"
+      end
+
+      def get(key)
+        data = @data[path_for(key)]
+        return data.value if data and not data.deleted?
+        nil
+      end
+      
+      def set(key, value)
+        key = path_for(key)
+        if not @data[key]
+          @data[key] = Data.new(key, value, DCell.id)
+        else
+          @data[key].value = value
+        end
+      end
+
+      def observe(other)
+        key = other.key
+        if not @data[key]
+          @data[key] = other
+          @data[key].observe
+          Celluloid::Logger.debug "Observed new data #{key} => #{other.value}"
+        else
+          @data[key].merge!(other)
+        end
+      end
+
+      def keys
+        @data.keys.map { |k| k =~ /#{@base_path}\/(.+)$/; $1 }
+      end
+
+      def clear
+        @data.map(&:clear)
+      end
+
+      def changed
+        @data.each_value.select(&:changed?)
+      end
+
+      def values
+        @data.values
+      end
+    end
+  end
+end

data/lib/dcell/registries/gossip_adapter.rb

@@ -0,0 +1,26 @@
+require 'forwardable'
+
+module DCell
+  module Registry
+    class GossipAdapter
+      extend Forwardable
+      PREFIX = "/dcell"
+
+      def_delegator  :@global_registry, :get,     :get_global
+      def_delegator  :@global_registry, :set,     :set_global
+      def_delegator  :@global_registry, :clear,   :clear_globals
+      def_delegator  :@global_registry, :keys,    :global_keys
+      def_delegators :@global_registry, :changed, :observe, :values
+
+      def initialize(options)
+        # Convert all options to symbols :/
+        options = options.inject({}) { |h,(k,v)| h[k.to_sym] = v; h }
+
+        @env = options[:env] || 'production'
+        @base_path = options[:namespace] || "#{PREFIX}/#{@env}"
+
+        @global_registry = Gossip::Store.new("#{@base_path}/globals")
+      end
+    end
+  end
+end

data/lib/dcell/registries/moneta_adapter.rb

@@ -16,7 +16,6 @@ module DCell
         # @moneta = Moneta::TieredCache.new options
         @moneta = Moneta::Memory.new options
 
-        @node_registry   = Registry.new(@moneta, :nodes)
         @global_registry = Registry.new(@moneta, :globals)
       end
 
@@ -39,7 +38,6 @@ module DCell
         end
 
         # DCell registry behaviors
-        alias_method :nodes, :all
         alias_method :global_keys, :all
 
         def clear
@@ -47,11 +45,6 @@ module DCell
         end
       end
 
-      def get_node(node_id);       @node_registry.get(node_id) end
-      def set_node(node_id, addr); @node_registry.set(node_id, addr) end
-      def nodes;                   @node_registry.nodes end
-      def clear_nodes;             @node_registry.clear end
-
       def get_global(key);        @global_registry.get(key) end
       def set_global(key, value); @global_registry.set(key, value) end
       def global_keys;            @global_registry.global_keys end