herdis 0.0.1

data/bin/herdis

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'bundler/setup'
+require File.expand_path(File.join(File.dirname(__FILE__), "..", "lib", "herdis", "server"))
+

data/lib/herdis/client.rb

@@ -0,0 +1,136 @@
+
+require 'hiredis'
+require 'redis'
+require 'redis/distributed'
+require 'yajl'
+require 'digest/sha1'
+require 'pp'
+
+$LOAD_PATH.unshift(File.expand_path('lib'))
+
+require 'herdis/common'
+
+module Herdis
+  
+  class Client
+
+    class ReDistributed < Redis::Distributed
+      
+      attr_reader :nodes
+
+      def initialize(urls, options = {})
+        @tag = options.delete(:tag) || /^\{(.+?)\}/
+        @default_options = options
+        @nodes = urls.map { |url| Redis.connect(options.merge(:url => url)) }
+        @subscribed_node = nil
+      end
+
+      def node_for(key)
+        @nodes[Digest::SHA1.hexdigest(key_tag(key.to_s) || key.to_s).to_i(16) % @nodes.size]
+      end
+      
+      def add_node(url)
+        raise "You can't add nodes to #{self}!"
+      end
+      
+    end
+    
+    class DeadClusterException < RuntimeError
+    end
+
+    attr_reader :options, :shepherds, :dredis
+
+    def initialize(*args)
+      @options = args.last.is_a?(Hash) ? args.pop : {}
+      @shepherds = {}
+      args.each_with_index do |url, index|
+        @shepherds["initial#{index}"] = {"url" => url}
+      end
+      begin
+        refresh_cluster
+      rescue DeadClusterException => e
+        raise "No such cluster: #{url}"
+      end
+    end
+
+    def create_urls(cluster)
+      hash = {}
+      cluster.each do |shepherd_id, shepherd_status|
+        shepherd_url = URI.parse(shepherd_status["url"])
+        (shepherd_status["masters"] || []).each do |shard_id|
+          hash[shard_id.to_i] = "redis://#{shepherd_url.host}:#{shepherd_status["first_port"].to_i + shard_id.to_i}/"
+        end
+      end
+      urls = hash.keys.sort.collect do |key|
+        hash[key]
+      end
+    end
+
+    def validate(urls)
+      unless urls.size == Herdis::Common::SHARDS
+        raise "Broken cluster, there should be #{Herdis::Common::SHARDS} shards, but are #{urls.size}"
+      end
+      creators = Set.new
+      urls.each_with_index do |url, index|
+        parsed = URI.parse(url)
+        r = Redis.new(:host => parsed.host, :port => parsed.port)
+        claimed_shard = r.get("Herdis::Shepherd::Shard.id").to_i
+        creators << r.get("Herdis::Shepherd::Shard.created_by")
+        raise "Broken cluster, shard #{index} claims to be shard #{claimed_shard}" unless claimed_shard == index
+        raise "Broken cluster, multiple creators: #{creators.inspect}" if creators.size > 1
+      end
+    end
+
+    def refresh_cluster
+      cluster = nil
+      while cluster.nil?
+        raise DeadClusterException.new if @shepherds.empty?
+        random_shepherd_id = @shepherds.keys[rand(@shepherds.size)]
+        cluster_request = 
+          EM::HttpRequest.new(@shepherds[random_shepherd_id]["url"]).get(:path => "/cluster",
+                                                                         :head => {"Accept" => "application/json"})
+        if cluster_request.response_header.status == 0
+          @shepherds.delete(random_shepherd_id)
+        else
+          cluster = Yajl::Parser.parse(cluster_request.response)
+          begin
+            urls = create_urls(cluster)
+            validate(urls)
+          rescue Errno::ECONNREFUSED => e
+            cluster = nil
+          rescue RuntimeError => e
+            if e.message == "ERR operation not permitted"
+              cluster = nil
+            else
+              raise e
+            end
+          end
+        end
+      end
+      @shepherds = cluster
+      @dredis = ReDistributed.new(urls,
+                                  @options)
+    end      
+    
+    def method_missing(meth, *args, &block)
+      begin
+        @dredis.send(meth, *args, &block)
+      rescue DeadClusterException => e
+        refresh_cluster
+        retry
+      rescue Errno::ECONNREFUSED => e
+        refresh_cluster
+        retry
+      rescue RuntimeError => e
+        if e.message == "ERR operation not permitted"
+          refresh_cluster
+          retry
+        else
+          raise e
+        end
+      end
+    end
+
+  end
+  
+end

data/lib/herdis/common.rb

@@ -0,0 +1,10 @@
+
+module Herdis
+
+  module Common
+
+    SHARDS = 128
+
+  end
+
+end

data/lib/herdis/handlers/add_shards.rb

@@ -0,0 +1,20 @@
+
+module Herdis
+
+   module Handlers
+
+    class AddShards < Goliath::API
+      include Common
+
+      def response(env)
+        shard_ids = env['params']["shard_ids"]
+        shepherd_id = env['params'][:shepherd_id]
+        Herdis::Plugins::ShepherdConnection.shepherd.add_shards(shepherd_id, shard_ids)
+        [201, {}, ""]
+      end
+      
+    end
+
+  end
+
+end

data/lib/herdis/handlers/add_shepherd.rb

@@ -0,0 +1,21 @@
+
+module Herdis
+
+   module Handlers
+
+    class AddShepherd < Goliath::API
+      include Common
+
+      def response(env)
+        data = env['params']
+        data.delete(:shepherd_id)
+        data.delete("shepherd_id")
+        Herdis::Plugins::ShepherdConnection.shepherd.add_shepherd(data)
+        [201, {}, ""]
+      end
+      
+    end
+
+  end
+
+end

data/lib/herdis/handlers/cluster.rb

@@ -0,0 +1,21 @@
+
+module Herdis
+
+   module Handlers
+
+    class Cluster < Goliath::API
+      include Common
+
+      def response(env)
+        if Herdis::Plugins::ShepherdConnection.shepherd.nil?
+          [404, {}, ""]
+        else
+          [200, {}, Herdis::Plugins::ShepherdConnection.shepherd.cluster_status]
+        end
+      end
+      
+    end
+
+  end
+
+end

data/lib/herdis/handlers/common.rb

@@ -0,0 +1,18 @@
+
+module Herdis
+  module Handlers
+    module Common
+      
+      def self.included(base)
+        base.use AsyncRack::Runtime
+        base.use AsyncRack::Deflater
+        base.use Goliath::Rack::DefaultMimeType
+        base.use Goliath::Rack::Formatters::JSON
+        base.use Goliath::Rack::Params
+        base.use Herdis::Rack::DefaultHeaders
+        base.use Herdis::Rack::HostParameter
+      end
+      
+    end
+  end
+end

data/lib/herdis/handlers/info.rb

@@ -0,0 +1,21 @@
+
+module Herdis
+
+   module Handlers
+
+    class Info < Goliath::API
+      include Common
+
+      def response(env)
+        if Herdis::Plugins::ShepherdConnection.shepherd.nil?
+          [404, {}, ""]
+        else
+          [200, {}, Herdis::Plugins::ShepherdConnection.shepherd.cluster_info]
+        end
+      end
+      
+    end
+
+  end
+
+end

data/lib/herdis/handlers/join_cluster.rb

@@ -0,0 +1,18 @@
+
+module Herdis
+
+   module Handlers
+
+    class JoinCluster < Goliath::API
+      include Common
+
+      def response(env)
+        Herdis::Plugins::ShepherdConnection.shepherd.join_cluster(env['params']['url'])
+        [204, {}, ""]
+      end
+      
+    end
+
+  end
+
+end

data/lib/herdis/handlers/ping.rb

@@ -0,0 +1,17 @@
+
+module Herdis
+
+   module Handlers
+
+    class Ping < Goliath::API
+      include Common
+
+      def response(env)
+        [Herdis::Plugins::ShepherdConnection.shepherd.status, {}, ""]
+      end
+      
+    end
+
+  end
+
+end

data/lib/herdis/handlers/remove_shards.rb

@@ -0,0 +1,20 @@
+
+module Herdis
+
+   module Handlers
+
+    class RemoveShards < Goliath::API
+      include Common
+
+      def response(env)
+        shard_ids = env['params']["shard_ids"]
+        shepherd_id = env['params'][:shepherd_id]
+        Herdis::Plugins::ShepherdConnection.shepherd.remove_shards(shepherd_id, shard_ids)
+        [204, {}, ""]
+      end
+      
+    end
+
+  end
+
+end

data/lib/herdis/handlers/remove_shepherd.rb

@@ -0,0 +1,18 @@
+
+module Herdis
+
+   module Handlers
+
+    class RemoveShepherd < Goliath::API
+      include Common
+
+      def response(env)
+        Herdis::Plugins::ShepherdConnection.shepherd.remove_shepherd(env['params'][:shepherd_id])
+        [204, {}, ""]
+      end
+      
+    end
+
+  end
+
+end

data/lib/herdis/handlers/sanity.rb

@@ -0,0 +1,21 @@
+
+module Herdis
+
+   module Handlers
+
+    class Sanity < Goliath::API
+      include Common
+
+      def response(env)
+        if Herdis::Plugins::ShepherdConnection.shepherd.nil?
+          [404, {}, ""]
+        else
+          [200, {}, Herdis::Plugins::ShepherdConnection.shepherd.sanity]
+        end
+      end
+      
+    end
+
+  end
+
+end

data/lib/herdis/handlers/shutdown.rb

@@ -0,0 +1,18 @@
+
+module Herdis
+
+   module Handlers
+
+    class Shutdown < Goliath::API
+      include Common
+
+      def response(env)
+        Herdis::Plugins::ShepherdConnection.shutdown
+        [200, {}, ""]
+      end
+      
+    end
+
+  end
+
+end

data/lib/herdis/plugins/shepherd_connection.rb

@@ -0,0 +1,49 @@
+module Herdis
+  module Plugins
+    class ShepherdConnection
+
+      @@shepherd = nil
+      
+      def self.shepherd
+        @@shepherd
+      end
+      
+      def self.shutdown
+        @@shepherd.shutdown unless @@shepherd.nil?
+        @@shepherd = nil
+      end
+      
+      def initialize(port, config, status, logger)
+        @port = port
+        @logger = logger
+      end
+
+      def run
+        opts = {}
+        copy_from_env(opts, :first_port, :to_i)
+        copy_from_env(opts, :dir)
+        copy_from_env(opts, :shepherd_id)
+        copy_from_env(opts, :inmemory)
+        copy_from_env(opts, :redundancy, :to_i)
+        copy_from_env(opts, :connect_to)
+        opts[:port] = @port
+        opts[:logger] = @logger
+        @@shepherd = Herdis::Shepherd.new(opts)
+      end
+      
+      private
+      
+      def copy_from_env(hash, key, *mutators)
+        env_key = key.to_s.upcase
+        env_key = "SHEPHERD_#{env_key}" unless env_key.index("SHEPHERD_") == 0
+        if ENV[env_key]
+          hash[key] = ENV[env_key] 
+          mutators.each do |mutator|
+            hash[key] = hash[key].send(mutator)
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/herdis/rack/default_headers.rb

@@ -0,0 +1,15 @@
+module Herdis
+  module Rack
+    class DefaultHeaders
+      include Goliath::Rack::AsyncMiddleware
+
+      DEFAULT_HEADERS = {
+        'Content-Type' => 'application/json'
+      }
+
+      def post_process(env, status, headers, body)
+        [status, headers.merge(DEFAULT_HEADERS), body]
+      end
+    end
+  end
+end

data/lib/herdis/rack/favicon.rb

@@ -0,0 +1,31 @@
+require 'time'
+
+# Reads a favicon.ico statically at load time, renders it on any request for
+# '/favicon.ico', and sends every other request on downstream.
+#
+# Rack::Static is a better option if you're serving several static assets.
+#
+module Herdis
+  module Rack
+    class Favicon
+      def initialize(app, filename)
+        @app = app
+        @favicon = File.read(File.join(filename))
+        @expires  = Time.at(Time.now + (60 * 60 * 24 * 7)).utc.rfc822.to_s
+        @last_modified = File.mtime(filename).utc.rfc822.to_s
+      end
+
+      def call(env)
+        if env['REQUEST_PATH'] == '/favicon.ico'
+          env.logger.info('Serving favicon.ico')
+
+          [200, {'Last-Modified' => @last_modified,
+                 'Expires' => @expires,
+                 'Content-Type' => "image/png"}, @favicon]
+        else
+          @app.call(env)
+        end
+      end
+    end
+  end
+end

data/lib/herdis/rack/host_parameter.rb

@@ -0,0 +1,22 @@
+
+module Herdis
+
+  module Rack
+
+    class HostParameter
+
+      include Goliath::Rack::AsyncMiddleware
+
+      def call(env)
+        class << Fiber.current
+          attr_accessor :host
+        end
+        Fiber.current.host = env["SERVER_NAME"]
+        super(env)
+      end
+
+    end
+
+  end
+
+end

data/lib/herdis/rmerge.rb

@@ -0,0 +1,74 @@
+# 
+# = Hash Recursive Merge
+# 
+# Merges a Ruby Hash recursively, Also known as deep merge.
+# Recursive version of Hash#merge and Hash#merge!.
+# 
+# Category::    Ruby
+# Package::     Hash
+# Author::      Simone Carletti <weppos@weppos.net>
+# Copyright::   2007-2008 The Authors
+# License::     MIT License
+# Link::        http://www.simonecarletti.com/
+# Source::      http://gist.github.com/gists/6391/
+#
+module HashRecursiveMerge
+
+  #
+  # Recursive version of Hash#merge!
+  # 
+  # Adds the contents of +other_hash+ to +hsh+, 
+  # merging entries in +hsh+ with duplicate keys with those from +other_hash+.
+  # 
+  # Compared with Hash#merge!, this method supports nested hashes.
+  # When both +hsh+ and +other_hash+ contains an entry with the same key,
+  # it merges and returns the values from both arrays.
+  # 
+  #    h1 = {"a" => 100, "b" => 200, "c" => {"c1" => 12, "c2" => 14}}
+  #    h2 = {"b" => 254, "c" => 300, "c" => {"c1" => 16, "c3" => 94}}
+  #    h1.rmerge!(h2)   #=> {"a" => 100, "b" => 254, "c" => {"c1" => 16, "c2" => 14, "c3" => 94}}
+  #    
+  # Simply using Hash#merge! would return
+  # 
+  #    h1.merge!(h2)    #=> {"a" => 100, "b" = >254, "c" => {"c1" => 16, "c3" => 94}}
+  # 
+  def rmerge!(other_hash)
+    merge!(other_hash) do |key, oldval, newval| 
+        oldval.class == self.class ? oldval.rmerge!(newval) : newval
+    end
+  end
+
+  #
+  # Recursive version of Hash#merge
+  # 
+  # Compared with Hash#merge!, this method supports nested hashes.
+  # When both +hsh+ and +other_hash+ contains an entry with the same key,
+  # it merges and returns the values from both arrays.
+  # 
+  # Compared with Hash#merge, this method provides a different approch
+  # for merging nasted hashes.
+  # If the value of a given key is an Hash and both +other_hash+ abd +hsh
+  # includes the same key, the value is merged instead replaced with
+  # +other_hash+ value.
+  # 
+  #    h1 = {"a" => 100, "b" => 200, "c" => {"c1" => 12, "c2" => 14}}
+  #    h2 = {"b" => 254, "c" => 300, "c" => {"c1" => 16, "c3" => 94}}
+  #    h1.rmerge(h2)    #=> {"a" => 100, "b" => 254, "c" => {"c1" => 16, "c2" => 14, "c3" => 94}}
+  #    
+  # Simply using Hash#merge would return
+  # 
+  #    h1.merge(h2)     #=> {"a" => 100, "b" = >254, "c" => {"c1" => 16, "c3" => 94}}
+  # 
+  def rmerge(other_hash)
+    r = {}
+    merge(other_hash)  do |key, oldval, newval| 
+      r[key] = oldval.class == self.class ? oldval.rmerge(newval) : newval
+    end
+  end
+
+end
+
+
+class Hash
+  include HashRecursiveMerge
+end

data/lib/herdis/server.rb

@@ -0,0 +1,59 @@
+
+require 'em-synchrony'
+require 'em-synchrony/em-http'
+require 'hiredis'
+require 'redis'
+require 'goliath'
+require 'yajl'
+require 'pp'
+
+$LOAD_PATH.unshift(File.expand_path('lib'))
+
+require 'herdis/rmerge'
+require 'herdis/common'
+require 'herdis/shepherd'
+require 'herdis/plugins/shepherd_connection'
+require 'herdis/rack/default_headers'
+require 'herdis/rack/favicon'
+require 'herdis/rack/host_parameter'
+require 'herdis/handlers/common'
+require 'herdis/handlers/cluster'
+require 'herdis/handlers/join_cluster'
+require 'herdis/handlers/remove_shepherd'
+require 'herdis/handlers/shutdown'
+require 'herdis/handlers/ping'
+require 'herdis/handlers/info'
+require 'herdis/handlers/sanity'
+require 'herdis/handlers/add_shepherd'
+require 'herdis/handlers/add_shards'
+require 'herdis/handlers/remove_shepherd'
+require 'herdis/handlers/remove_shards'
+
+module Herdis
+
+  class Server < Goliath::API
+
+    plugin Herdis::Plugins::ShepherdConnection
+    use Herdis::Rack::Favicon, File.join(File.dirname(__FILE__), "..", "..", "assets", "shepherd.png")
+    
+    head '/', Herdis::Handlers::Ping
+
+    get '/', Herdis::Handlers::Info
+    get '/cluster', Herdis::Handlers::Cluster
+    get '/sanity', Herdis::Handlers::Sanity
+
+    post '/', Herdis::Handlers::JoinCluster
+
+    post '/:shepherd_id/shards', Herdis::Handlers::AddShards
+    delete '/:shepherd_id/shards', Herdis::Handlers::RemoveShards
+
+    put '/:shepherd_id', Herdis::Handlers::AddShepherd
+    delete '/:shepherd_id', Herdis::Handlers::RemoveShepherd
+
+    delete '/', Herdis::Handlers::Shutdown
+  end
+
+end
+
+Goliath::Application.app_class = Herdis::Server
+

data/lib/herdis/shepherd.rb

@@ -0,0 +1,531 @@
+
+module Herdis
+
+  class Shepherd
+
+    CHECK_SLAVE_TIMER = (ENV["SHEPHERD_CHECK_SLAVE_TIMER"] || 10).to_f
+    CHECK_PREDECESSOR_TIMER = (ENV["SHEPHERD_CHECK_PREDECESSOR_TIMER"] || 1).to_f
+
+    class Shard
+
+      attr_reader :shepherd
+      attr_reader :id
+      attr_reader :master
+      def initialize(options = {})
+        @shepherd = options.delete(:shepherd)
+        @id = options.delete(:id)
+        @master = options.delete(:master)
+        Dir.mkdir(dir) unless Dir.exists?(dir)
+        initialize_redis
+      end
+      def dir
+        File.join(shepherd.dir, "shard#{id}")
+      end
+      def port
+        shepherd.first_port + id.to_i
+      end
+      def inmemory
+        shepherd.inmemory
+      end
+      def connection
+        if master
+          @connection ||= Redis.new(:host => "localhost", :port => port, :password => "slaved")
+        else
+          @connection ||= Redis.new(:host => "localhost", :port => port)
+        end
+      end
+      def inspect
+        begin
+          super
+        rescue Errno::ECONNREFUSED => e
+          "#<#{self.class.name} @dir=#{dir} @port=#{port} CLOSED>"
+        end
+      end
+      def liberate!
+        if master
+          @master = nil
+          begin
+            connection.slaveof("NO", "ONE")
+            connection.config("set", "requirepass", "")
+            shepherd.slaves.delete(id.to_s)
+            shepherd.masters[id.to_s] = self
+          rescue RuntimeError => e
+            if e.message == "LOADING Redis is loading the dataset in memory"
+              EM::Synchrony.sleep(0.1)
+            else
+              raise e
+            end
+          end
+        end
+      end
+      def enslave!(external_uri)
+        unless external_uri == master
+          connection.shutdown
+          @master = external_uri
+          @connection = nil
+          initialize_redis
+          shepherd.masters.delete(id.to_s)
+          shepherd.slaves[id.to_s] = self
+        end
+      end
+      def initialize_redis
+        begin
+          begin
+            connection.shutdown          
+          rescue RuntimeError => e
+            if e.message == "ERR Client sent AUTH, but no password is set"
+              Redis.new(:host => "localhost", :port => port).shutdown
+            else
+              raise e
+            end
+          end
+        rescue Errno::ECONNREFUSED => e
+        end
+        io = IO.popen("#{shepherd.redis} -", "w")
+        write_configuration(io)
+        unless master
+          initialization = Proc.new do |p|
+            begin
+              connection.set("#{self.class.name}.id", id)
+              connection.set("#{self.class.name}.created_at", Time.now.to_i)
+              connection.set("#{self.class.name}.created_by", shepherd.shepherd_id)
+            rescue Errno::ECONNREFUSED => e
+              EM.add_timer(0.1) do
+                p.call(p)
+              end
+            end
+          end
+          EM.add_timer(0.1) do
+            initialization.call(initialization)
+          end
+        end
+      end
+      def write_configuration(io)
+        io.puts("daemonize yes")
+        io.puts("pidfile #{dir}/pid")
+        io.puts("port #{port}")
+        io.puts("timeout 300")
+        if master
+          io.puts("slaveof #{master.host} #{master.port}")
+          io.puts("requirepass slaved")
+        end
+        unless inmemory
+          io.puts("save 900 1")
+          io.puts("save 300 10")
+          io.puts("save 60 10000")
+          io.puts("dbfilename dump.rdb")
+        end
+        io.puts("dir #{dir}")
+        io.puts("logfile stdout")
+        io.close
+      end
+    end
+
+    attr_reader :dir
+    attr_reader :redis
+    attr_reader :first_port
+    attr_reader :inmemory
+    attr_reader :shepherd_id
+    attr_reader :redundancy
+    attr_reader :port
+    attr_reader :logger
+
+    attr_reader :masters
+    attr_reader :slaves
+    attr_reader :shepherds
+
+    def initialize(options = {})
+      @dir = options.delete(:dir) || File.join(ENV["HOME"], ".herdis")
+      @redis = options.delete(:redis) || "redis-server"
+      @port = options.delete(:port) || 9000
+      @logger = options.delete(:logger)
+      @first_port = options.delete(:first_port) || 9080
+      @inmemory = options.delete(:inmemory)
+      @redundancy = options.delete(:redundancy) || 2
+      @shepherd_id = options.delete(:shepherd_id) || rand(1 << 256).to_s(36)
+      Dir.mkdir(dir) unless Dir.exists?(dir)
+
+      @shepherds = {}
+      @slaves = {}
+      @masters = {}
+
+      at_exit do
+        shutdown
+      end
+
+      if connect_to = options.delete(:connect_to)
+        join_cluster(connect_to)
+      else
+        Herdis::Common::SHARDS.times do |shard_id|
+          create_master_shard(shard_id)
+        end
+        @shepherds[shepherd_id] = shepherd_status
+      end
+    end
+
+    def ensure_slave_check
+      @check_slave_timer ||= EM.add_periodic_timer(CHECK_SLAVE_TIMER) do
+        check_slaves
+      end
+    end
+
+    def ensure_predecessor_check
+      @check_predecessor_timer ||= EM.add_periodic_timer(CHECK_PREDECESSOR_TIMER) do
+        check_predecessor
+      end
+    end
+
+    def to_each_sibling(method, options, &block)
+      default_options = {:head => {"Content-Type" => "application/json"}}
+      multi = EM::Synchrony::Multi.new
+      shepherds.each do |shepherd_id, shepherd|
+        unless shepherd_id == self.shepherd_id
+          multi.add(shepherd_id, 
+                    EM::HttpRequest.new(shepherd["url"]).send(method, 
+                                                              default_options.rmerge(options)))
+        end
+      end
+      yield
+      Fiber.new do
+        multi.perform while !multi.finished?
+      end.resume
+    end
+
+    def join_cluster(url)
+      shutdown
+      @shepherds = Yajl::Parser.parse(EM::HttpRequest.new(url).get(:path => "/cluster",
+                                                                   :head => {"Content-Type" => "application/json"}).response)
+      add_shepherd(shepherd_status)
+    end
+
+    def add_shepherd(shepherd_status)
+      unless shepherd_status == shepherds[shepherd_status["id"]]
+        shepherds[shepherd_status["id"]] = shepherd_status
+        to_each_sibling(:aput,
+                        :path => "/#{shepherd_status["id"]}",
+                        :body => Yajl::Encoder.encode(shepherd_status)) do
+          check_shards
+        end
+      end
+    end
+
+    def remove_shepherd(shepherd_id)
+      if shepherds.include?(shepherd_id)
+        shepherds.delete(shepherd_id)
+        to_each_sibling(:adelete,
+                        :path => "/#{shepherd_id}") do
+          check_shards
+        end
+      end
+    end
+
+    def add_shards(shepherd_id, shard_ids, check = true)
+      if shepherd_state = shepherds[shepherd_id]
+        shard_id_set = Set.new(shard_ids)
+        current_shard_id_set = Set.new(shepherd_state["masters"])
+        unless current_shard_id_set.superset?(shard_id_set)
+          shepherd_state["masters"] = (current_shard_id_set | shard_id_set).to_a
+          to_each_sibling(:apost,
+                          :path => "/#{shepherd_id}/shards",
+                          :body => Yajl::Encoder.encode(:shard_ids => shard_ids.to_a)) do
+            check_shards if check
+          end
+        end
+      end
+    end
+
+    def remove_shards(shepherd_id, shard_ids, check = true)
+      if shepherd_state = shepherds[shepherd_id]
+        shard_id_set = Set.new(shard_ids)
+        current_shard_id_set = Set.new(shepherd_state["masters"])
+        unless (shard_id_set & current_shard_id_set).empty?
+          shepherd_state["masters"] = (current_shard_id_set - shard_id_set).to_a
+          to_each_sibling(:adelete,
+                          :path => "/#{shepherd_id}/shards",
+                          :body => Yajl::Encoder.encode(:shard_ids => shard_ids.to_a)) do
+            check_shards if check
+          end
+        end
+      end
+    end
+    
+    def host
+      @host ||= Fiber.current.host if Fiber.current.respond_to?(:host)
+      @host || "localhost"
+    end
+
+    def url
+      "http://#{host}:#{port}"
+    end
+
+    def shepherd_status
+      {
+        "url" => url,
+        "id" => shepherd_id,
+        "first_port" => first_port,
+        "masters" => masters.keys
+      }
+    end
+
+    def cluster_info
+      {
+        "shepherd_id" => shepherd_id,
+        "ordinal" => ordinal,
+        "slaves" => slaves.keys,
+        "masters" => masters.keys,
+        "redundancy" => redundancy,
+        "siblings" => shepherds.keys.sort,
+        "inmemory" => inmemory,
+        "check_slave_timer" => CHECK_SLAVE_TIMER,
+        "check_predecessor_timer" => CHECK_PREDECESSOR_TIMER,
+        "sanity" => "#{url}/sanity",
+        "cluster" => "#{url}/cluster"
+      }
+    end
+
+    def sanity
+      creators = Set.new
+      min_created_at = nil
+      max_created_at = nil
+      masters = 0
+      slaves = 0
+      consistent = true
+      urls = []
+      shard_status.each do |shard_url|
+        urls << shard_url
+        url = URI.parse(shard_url)
+        shard_connection = Redis.new(:host => url.host, :port => url.port)
+        info = shard_connection.info
+        masters += 1 if info["role"] == "master"
+        slaves += 1 if info["role"] == "slave"
+        created_at = shard_connection.get("#{Herdis::Shepherd::Shard.name}.created_at").to_i
+        min_created_at = created_at if min_created_at.nil? || created_at < min_created_at
+        max_created_at = created_at if max_created_at.nil? || created_at > max_created_at
+        creators << shard_connection.get("#{Herdis::Shepherd::Shard.name}.created_by")
+      end
+      {
+        :creators => creators.to_a,
+        :consistent => creators.size == 1 && masters == Herdis::Common::SHARDS && slaves == 0,
+        :min_created_at => Time.at(min_created_at).to_s,
+        :max_created_at => Time.at(max_created_at).to_s,
+        :masters => masters,
+        :slaves => slaves,
+        :shards => urls
+      }
+    end
+
+    def shard_status
+      rval = []
+      cluster_status.each do |shepherd_id, shepherd_status|
+        shepherd_status["masters"].each do |shard_id|
+          if rval[shard_id.to_i].nil?
+            rval[shard_id.to_i] = "redis://#{host}:#{shard_id.to_i + shepherd_status["first_port"].to_i}/"
+          else
+            raise Goliath::Validation::InternalServerError.new("Duplicate masters: #{shard_id}")
+          end
+        end
+      end
+      rval
+    end
+
+    def cluster_status
+      shepherds.merge(shepherd_id => shepherd_status)
+    end
+
+    def shutdown_shard(shard_id)
+      if shard = masters[shard_id.to_s]
+        shard.connection.shutdown
+        masters.delete(shard_id.to_s)
+      end
+    end
+
+    def shutdown_slave(shard_id)
+      if shard = slaves[shard_id.to_s]
+        shard.connection.shutdown
+        slaves.delete(shard_id.to_s)
+      end
+    end
+
+    def shutdown
+      masters.keys.each do |shard_id|
+        shutdown_shard(shard_id)
+      end
+      slaves.keys.each do |shard_id|
+        shutdown_slave(shard_id)
+      end
+    end
+
+    def create_external_shards(should_be_owned)
+      shepherds.values.inject({}) do |sum, shepherd_status|
+        if shepherd_status["id"] == shepherd_id
+          sum
+        else
+          shepherd_url = URI.parse(shepherd_status["url"])
+          sum.merge(shepherd_status["masters"].inject({}) do |sum, shard_id|
+                      shepherd_url = URI.parse(shepherd_status["url"])
+                      sum.merge(shard_id.to_s => URI.parse("redis://#{shepherd_url.host}:#{shepherd_status["first_port"].to_i + shard_id.to_i}/"))
+                    end)
+        end    
+      end
+    end
+
+    def check_shards
+      should_be_owned = owned_shards
+      should_be_backed_up = backup_shards
+      master_ids = Set.new(masters.keys)
+      slave_ids = Set.new(slaves.keys)
+      external_shards = create_external_shards(should_be_owned)
+      externally_running = Set.new(external_shards.keys)
+
+      needs_to_be_liberated = slave_ids - externally_running
+      needs_to_be_enslaved = (master_ids & externally_running & should_be_backed_up) - should_be_owned
+      needs_to_be_directed = slave_ids & externally_running & (should_be_backed_up | should_be_owned)
+      slaves_needing_to_be_shut_down = (slave_ids & externally_running) - (should_be_backed_up | should_be_owned)
+      masters_needing_to_be_shut_down = (master_ids & externally_running) - (should_be_backed_up | should_be_owned)
+      new_slaves_needed = ((should_be_backed_up | should_be_owned) & externally_running) - (slave_ids | master_ids)
+
+      handled = Set.new
+
+      logger.debug "#{shepherd_id} *** liberating #{needs_to_be_liberated.inspect}" unless needs_to_be_liberated.empty?
+      needs_to_be_liberated.each do |shard_id|
+        handled.add(shard_id.to_s)
+        slaves[shard_id.to_s].liberate!
+      end
+      add_shards(shepherd_id, needs_to_be_liberated, false)
+
+      logger.debug "#{shepherd_id} *** enslaving #{needs_to_be_enslaved.inspect}" unless needs_to_be_enslaved.empty?
+      needs_to_be_enslaved.each do |shard_id|
+        raise "Already liberated #{shard_id}!" if handled.include?(shard_id.to_s)
+        handled.add(shard_id.to_s)
+        masters[shard_id.to_s].enslave!(external_shards[shard_id.to_s])
+      end
+      remove_shards(shepherd_id, needs_to_be_enslaved, false)
+
+      logger.debug "#{shepherd_id} *** redirecting #{needs_to_be_directed.inspect}" unless needs_to_be_directed.empty?
+      needs_to_be_directed.each do |shard_id|
+        raise "Already liberated or enslaved #{shard_id}!" if handled.include?(shard_id.to_s)
+        handled.add(shard_id.to_s)
+        slaves[shard_id.to_s].enslave!(external_shards[shard_id.to_s])
+      end
+
+      logger.debug "#{shepherd_id} *** killing masters #{masters_needing_to_be_shut_down.inspect}" unless masters_needing_to_be_shut_down.empty?
+      masters_needing_to_be_shut_down.each do |shard_id|
+        raise "Already liberated, enslaved or directed #{shard_id}!" if handled.include?(shard_id.to_s)
+        handled.add(shard_id.to_s)
+        shutdown_shard(shard_id)
+      end
+      remove_shards(shepherd_id, masters_needing_to_be_shut_down, false)
+      
+      logger.debug "#{shepherd_id} *** killing slaves #{slaves_needing_to_be_shut_down.inspect}" unless slaves_needing_to_be_shut_down.empty?
+      slaves_needing_to_be_shut_down.each do |shard_id|
+        raise "Already liberated, enslaved, directed or shut down #{shard_id}!" if handled.include?(shard_id.to_s)
+        handled.add(shard_id.to_s)
+        shutdown_slave(shard_id)
+      end
+
+      logger.debug "#{shepherd_id} *** creating slaves #{new_slaves_needed.inspect}" unless new_slaves_needed.empty?
+      new_slaves_needed.each do |shard_id|
+        raise "Already liberated, enslaved, directed or shut down #{shard_id}!" if handled.include?(shard_id.to_s)
+        handled.add(shard_id.to_s)
+        create_slave_shard(shard_id.to_s, external_shards[shard_id.to_s])
+      end
+      
+      ensure_slave_check
+      ensure_predecessor_check
+    end
+    
+    def check_slaves
+      revolution = Set.new
+      (owned_shards & slaves.keys).each do |shard_id|
+        shard = slaves[shard_id.to_s]
+        if shard.connection.info["master_sync_in_progress"] == "0"
+          revolution << shard_id.to_s
+        end
+      end
+      unless revolution.empty?
+        logger.debug "#{shepherd_id} *** revolting #{revolution.inspect}"
+        add_shards(shepherd_id, revolution.to_a)
+      end
+    end
+
+    def create_master_shard(shard_id)
+      masters[shard_id.to_s] = create_shard(shard_id)
+    end
+
+    def status
+      204
+    end
+
+    def ordered_shepherd_keys
+      shepherds.keys.sort
+    end
+
+    def ordinal
+      shepherds.keys.sort.index(shepherd_id)
+    end
+
+    def backup_ordinals
+      rval = Set.new
+      ordered_keys = ordered_shepherd_keys
+      my_index = ordered_keys.index(shepherd_id)
+      [redundancy, ordered_keys.size - 1].min.times do |n|
+        rval << (my_index - n - 1) % ordered_keys.size
+      end
+      rval
+    end
+
+    def backup_shards
+      rval = Set.new
+      backup_ordinals.each do |ordinal|
+        rval += shards_owned_by(ordinal)
+      end
+      rval
+    end
+
+    def shards_owned_by(ordinal)
+      rval = Set.new
+      while ordinal < Herdis::Common::SHARDS
+        rval << ordinal.to_s
+        ordinal += shepherds.size
+      end
+      rval
+    end
+
+    def owned_shards
+      shards_owned_by(ordinal)
+    end
+
+    def predecessor
+      ordered_keys = ordered_shepherd_keys
+      my_index = ordered_keys.index(shepherd_id)
+      if my_index == 0
+        shepherds[ordered_keys.last]
+      else
+        shepherds[ordered_keys[my_index - 1]]
+      end
+    end
+
+    def check_predecessor
+      pre = predecessor
+      if pre && pre["id"] != shepherd_id
+        Fiber.new do
+          if EM::HttpRequest.new(pre["url"]).head.response_header.status != 204
+            logger.warn("#{shepherd_id} *** dropping #{pre["id"]} due to failure to respond to ping")
+            remove_shepherd(pre["id"])
+          end
+        end.resume
+      end
+    end
+
+    def create_slave_shard(shard_id, external_uri)
+      slaves[shard_id.to_s] = create_shard(shard_id, :master => external_uri)
+    end
+
+    def create_shard(shard_id, options = {})
+      Shard.new(options.merge(:shepherd => self,
+                              :id => shard_id))
+    end
+
+  end
+
+end

metadata

@@ -0,0 +1,136 @@
+--- !ruby/object:Gem::Specification
+name: herdis
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Martin Bruse
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-03-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: hiredis
+  requirement: &70331042073380 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70331042073380
+- !ruby/object:Gem::Dependency
+  name: em-synchrony
+  requirement: &70331042072880 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70331042072880
+- !ruby/object:Gem::Dependency
+  name: em-http-request
+  requirement: &70331042072400 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70331042072400
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: &70331042071940 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70331042071940
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: &70331042071440 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70331042071440
+- !ruby/object:Gem::Dependency
+  name: goliath
+  requirement: &70331042071020 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70331042071020
+description: ! 'A Redis herder for simplifying Redis presharding
+
+'
+email: zondolfin at gmail dot com
+executables:
+- herdis
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/herdis/client.rb
+- lib/herdis/common.rb
+- lib/herdis/handlers/add_shards.rb
+- lib/herdis/handlers/add_shepherd.rb
+- lib/herdis/handlers/cluster.rb
+- lib/herdis/handlers/common.rb
+- lib/herdis/handlers/info.rb
+- lib/herdis/handlers/join_cluster.rb
+- lib/herdis/handlers/ping.rb
+- lib/herdis/handlers/remove_shards.rb
+- lib/herdis/handlers/remove_shepherd.rb
+- lib/herdis/handlers/sanity.rb
+- lib/herdis/handlers/shutdown.rb
+- lib/herdis/plugins/shepherd_connection.rb
+- lib/herdis/rack/default_headers.rb
+- lib/herdis/rack/favicon.rb
+- lib/herdis/rack/host_parameter.rb
+- lib/herdis/rmerge.rb
+- lib/herdis/server.rb
+- lib/herdis/shepherd.rb
+- bin/herdis
+homepage: http://github.com/zond/herdis
+licenses: []
+post_install_message: 
+rdoc_options:
+- --line-numbers
+- --inline-source
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.15
+signing_key: 
+specification_version: 3
+summary: A Redis herder for simplifying Redis presharding
+test_files: []