redis_ring 0.0.2 → 0.1.0

data/.gitignore

@@ -1,4 +1,7 @@
 pkg/*
+tmp/*
+coverage/*
 *.gem
 .bundle
 .rvmrc
+tags

data/Gemfile.lock

@@ -1,10 +1,11 @@
 PATH
   remote: .
   specs:
-    redis_ring (0.0.1)
+    redis_ring (0.0.2)
       daemons-mikehale
       json
       sinatra
+      zookeeper
 
 GEM
   remote: http://rubygems.org/
@@ -13,7 +14,7 @@ GEM
     diff-lcs (1.1.2)
     json (1.5.1)
     mocha (0.9.12)
-    rack (1.2.1)
+    rack (1.2.2)
     rspec (2.5.0)
       rspec-core (~> 2.5.0)
       rspec-expectations (~> 2.5.0)
@@ -22,10 +23,14 @@ GEM
     rspec-expectations (2.5.0)
       diff-lcs (~> 1.1.2)
     rspec-mocks (2.5.0)
-    sinatra (1.2.0)
+    simplecov (0.4.1)
+      simplecov-html (~> 0.4.3)
+    simplecov-html (0.4.3)
+    sinatra (1.2.1)
       rack (~> 1.1)
       tilt (>= 1.2.2, < 2.0)
     tilt (1.2.2)
+    zookeeper (0.4.3)
 
 PLATFORMS
   ruby
@@ -34,3 +39,4 @@ DEPENDENCIES
   mocha
   redis_ring!
   rspec
+  simplecov

data/config/redis.conf.erb

@@ -20,18 +20,16 @@ loglevel notice
 
 databases 2048
 
-save 900 1
-save 300 10
-save 60 10000
+# save 900 1
+# save 300 10
+# save 60 10000
 
 rdbcompression yes
 
 # maxclients 128
 # maxmemory <bytes>
 
-appendonly no
-
-# The name of the append only file (default: "appendonly.aof")
+appendonly yes
 
 # appendfsync always
 appendfsync everysec

data/lib/redis_ring/application.rb

@@ -2,50 +2,50 @@ module RedisRing
 
   class Application
 
-    attr_reader :shards, :configuration, :process_manager
+    attr_reader :shards, :configuration, :process_manager, :zookeeper_observer, :master, :slave, :zookeeper_connection, :master_rpc, :http_client, :node_provider, :slave_rpc
 
-    def initialize(configuration)
-      @configuration = configuration
+    def initialize(config)
+      @configuration = config
       @process_manager = ProcessManager.new
-      @shards = {}
+      @http_client = HttpClient.new
+      @master_rpc = MasterRPC.new(http_client)
+      @slave_rpc = SlaveRPC.new(http_client)
+      @node_provider = NodeProvider.new(slave_rpc)
+      @zookeeper_connection = ZookeeperConnection.new(config.host_name,
+                                                      config.base_port,
+                                                      config.zookeeper_address)
+      @master = Master.new(zookeeper_connection, config.ring_size, node_provider)
+      @slave = Slave.new(configuration, master_rpc, process_manager)
+      @zookeeper_observer = ZookeeperObserver.new(zookeeper_connection, master, slave)
+      @web_interface_runner = WebInterfaceRunner.new(config.base_port, master, slave)
     end
 
     def start
       self.stop
 
-      @configuration.ring_size.times do |shard_number|
-        shard_conf = ShardConfig.new(shard_number, configuration)
-        @shards[shard_number] = Shard.new(shard_conf)
-      end
-
-      @shards.each do |shard_no, shard|
-        @process_manager.start_shard(shard)
-      end
+      @web_thread = @web_interface_runner.run
 
-      @process_manager.run
-    end
+      @zookeeper_connection.connect
+      @slave.node_id = @zookeeper_connection.current_node
 
-    def stop
-      @process_manager.halt
+      @zookeeper_thread = @zookeeper_observer.run
+      @pm_thread = @process_manager.run
 
-      @shards.each do |shard_no, shard|
-        @process_manager.stop_shard(shard)
+      [:INT, :TERM, :QUIT].each do |sig|
+        trap(sig) { self.stop }
       end
-
-      @shards = {}
     end
 
-    def shards_hash
-      shards_hash = {}
-      shards.each do |shard_no, shard|
-        shards_hash[shard_no] = { :host => shard.host, :port => shard.port, :status => shard.status }
-      end
-
-      return { :count => configuration.ring_size, :shards => shards_hash }
+    def wait
+      @pm_thread.join if @pm_thread
+      @zookeeper_thread.join if @zookeeper_thread
+      @web_thread.join if @web_thread
     end
 
-    class << self
-      attr_accessor :instance
+    def stop
+      @process_manager.halt
+      @zookeeper_observer.halt
+      @web_interface_runner.halt
     end
 
   end

data/lib/redis_ring/background_thread.rb

@@ -0,0 +1,45 @@
+module RedisRing
+
+  module BackgroundThread
+
+    def before_run
+    end
+
+    def after_halt
+    end
+
+    def do_work
+    end
+
+    def run
+      before_run
+
+      @continue_running = true
+
+      return Thread.new do
+        begin
+          while continue_running?
+            do_work
+          end
+          after_halt
+        rescue SystemExit
+          raise
+        rescue => e
+          puts "Error caught in #{self.class.name}:"
+          puts e
+          puts e.backtrace.join("\n")
+        end
+      end
+    end
+
+    def continue_running?
+      @continue_running
+    end
+
+    def halt
+      @continue_running = false
+    end
+
+  end
+
+end

data/lib/redis_ring/cli.rb

@@ -42,12 +42,9 @@ USAGE
     def start(config_file = nil)
       config = config_file ? Configuration.from_yml_file(config_file) : Configuration.new
 
-      Application.instance = Application.new(config)
-      Application.instance.start
-
-      WebInterface.run!(:port => config.base_port)
-
-      Application.instance.stop
+      app = Application.new(config)
+      app.start
+      app.wait
     end
 
   end

data/lib/redis_ring/configuration.rb

@@ -7,7 +7,7 @@ module RedisRing
   class Configuration
 
     PARAMETERS = [:host_name, :base_port, :ring_size, :redis_path, :redis_config_template_path,
-      :total_vm_size, :base_directory, :password, :total_max_memory, :vm_page_size]
+      :total_vm_size, :base_directory, :password, :total_max_memory, :vm_page_size, :zookeeper_address]
 
     attr_reader *PARAMETERS
 
@@ -50,6 +50,7 @@ module RedisRing
       self.base_directory ||= "/var/lib/redis"
       self.total_max_memory ||= 1024 * 1024 * 1024 # 1GB
       self.vm_page_size ||= 32
+      self.zookeeper_address ||= "localhost:2181"
     end
 
     def validate!

data/lib/redis_ring/http_client.rb

@@ -0,0 +1,23 @@
+module RedisRing
+
+  class HttpClient
+
+    def get(host, port, path, params = {})
+      Net::HTTP.get(uri(host, port, path, params))
+    end
+
+    def post(host, port, path, params = {})
+      Net::HTTP.post_form(uri(host, port, path, params), {}).body
+    end
+
+    protected
+
+    def uri(host, port, path, params)
+      params_str = params.map{|k,v| "#{k}=#{v}"}.join("&")
+      params_str = "?" + params_str unless params_str.empty?
+      URI.parse("http://#{host}:#{port}#{path}#{params_str}")
+    end
+
+  end
+
+end

data/lib/redis_ring/master.rb

@@ -0,0 +1,154 @@
+module RedisRing
+
+  class Master
+
+    attr_reader :zookeeper_connection, :ring_size, :node_provider
+
+    def initialize(zookeeper_connection, ring_size, node_provider)
+      @zookeeper_connection = zookeeper_connection
+      @ring_size = ring_size
+      @node_provider = node_provider
+      @node_ids = []
+      @is_master = false
+    end
+
+    def became_master
+      return if is_master?
+
+      puts "BECAME MASTER"
+
+      @is_master = true
+    end
+
+    def no_longer_is_master
+      return unless is_master?
+
+      puts "LOST MASTER STATUS"
+
+      @is_master = false
+    end
+
+    def nodes_changed(changed_node_ids)
+      return unless is_master?
+
+      new_nodes = changed_node_ids - node_ids
+      removed_nodes = node_ids - changed_node_ids
+
+      puts "NODES CHANGED"
+      puts "NEW: #{new_nodes.join(", ")}" if new_nodes.any?
+      puts "REMOVED: #{removed_nodes.join(', ')}" if removed_nodes.any?
+
+      @node_ids = changed_node_ids
+
+      reassign_shards
+    end
+
+    def node_joined(node_id)
+      puts "NODE JOINED #{node_id}"
+
+      reassign_shards
+    end
+
+    def node_leaving(node_id)
+      puts "NODE LEAVING #{node_id}"
+
+      node_ids.delete(node_id)
+      reassign_shards
+    end
+
+    def is_master?
+      return @is_master
+    end
+
+    def reassign_shards
+      update_node_statuses
+
+      running_shards = {}
+      best_candidates = {}
+      best_candidates_timestamps = Hash.new(0)
+
+      nodes.each do |node_id, node|
+        node.running_shards.dup.each do |shard_no|
+          if running_shards.key?(shard_no)
+            node.stop_shard(shard_no)
+          else
+            running_shards[shard_no] = node_id
+          end
+        end
+
+        node.available_shards.each do |shard_no, timestamp|
+          if timestamp > best_candidates_timestamps[shard_no]
+            best_candidates[shard_no] = node_id
+            best_candidates_timestamps[shard_no] = timestamp
+          end
+        end
+      end
+
+      offline_shards = (0...ring_size).to_a - running_shards.keys
+      shards_per_node = (1.0 * ring_size / nodes.size).floor
+      rest = ring_size - shards_per_node * nodes.size
+
+      nodes.each do |node_id, node|
+        next unless node.joined?
+        break if offline_shards.empty?
+        count_to_assign = shards_per_node - node.running_shards.size
+        count_to_assign += 1 if node_ids.index(node_id) < rest
+        count_to_assign.times do
+          shard_no = offline_shards.shift
+          break unless shard_no
+          node.start_shard(shard_no)
+        end
+      end
+
+      zookeeper_connection.update_status(status)
+    end
+
+    def status
+      {
+        :ring_size => ring_size,
+        :shards => shards
+      }
+    end
+
+    protected
+
+    attr_reader :node_ids
+    attr_accessor :nodes
+
+    def shards
+      running_shards = {}
+      nodes.each do |node_id, node|
+        node.running_shards.each do |shard_no|
+          running_shards[shard_no] = {
+            :host => node.host,
+            :port => node.port + shard_no + 1,
+            :status => :running
+          }
+        end
+      end
+      return running_shards
+    end
+
+    def update_node_statuses
+      self.nodes ||= {}
+
+      nodes.each do |node_id, node|
+        unless node_ids.include?(node_id)
+          nodes.delete(node_id)
+        end
+      end
+
+      node_ids.each do |node_id|
+        next if nodes.key?(node_id)
+        node_data = zookeeper_connection.node_data(node_id)
+        nodes[node_id] = node_provider.new(node_data["host"], node_data["port"])
+      end
+
+      nodes.each do |node_id, node|
+        node.update_status!
+      end
+    end
+
+  end
+
+end

data/lib/redis_ring/master_rpc.rb

@@ -0,0 +1,33 @@
+module RedisRing
+
+  class MasterRPC
+
+    attr_reader :http_client
+
+    def initialize(http_client)
+      @http_client = http_client
+    end
+
+    def connection(host, port)
+      Connection.new(http_client, host, port)
+    end
+
+    class Connection
+
+      attr_reader :http_client, :host, :port
+
+      def initialize(http_client, host, port)
+        @http_client = http_client
+        @host = host
+        @port = port
+      end
+
+      def node_loaded(node_id)
+        http_client.post(host, port, "/master/node_joined/#{node_id}")
+      end
+
+    end
+
+  end
+
+end

data/lib/redis_ring/node.rb

@@ -0,0 +1,66 @@
+module RedisRing
+
+  class NodeProvider
+
+    attr_reader :slave_rpc
+
+    def initialize(slave_rpc)
+      @slave_rpc = slave_rpc
+    end
+
+    def new(host, port)
+      Node.new(slave_rpc.connection(host, port), host, port)
+    end
+
+  end
+
+  class Node
+
+    attr_reader :slave_rpc, :host, :port
+
+    def initialize(slave_rpc, host, port)
+      @slave_rpc = slave_rpc
+      @host = host
+      @port = port
+    end
+
+    def update_status!
+      status_hash = slave_rpc.status
+      @joined = status_hash["joined"]
+      @running_shards = status_hash["running_shards"] || []
+      @available_shards = keys_to_i(status_hash["available_shards"] || {})
+    end
+
+    def joined?
+      @joined
+    end
+
+    def start_shard(shard_number)
+      running_shards << shard_number
+      slave_rpc.start_shard(shard_number)
+    end
+
+    def stop_shard(shard_number)
+      running_shards.delete(shard_number)
+      slave_rpc.stop_shard(shard_number)
+    end
+
+    def running_shards
+      @running_shards ||= []
+    end
+
+    def available_shards
+      @available_shards ||= {}
+    end
+
+    protected
+
+    def keys_to_i(hash)
+      result = {}
+      hash.each { |key, val| result[key.to_i] = val }
+      return result
+    end
+
+  end
+
+end

data/lib/redis_ring/process_manager.rb

@@ -4,49 +4,63 @@ module RedisRing
 
   class ProcessManager
 
+    include RedisRing::BackgroundThread
+
     def initialize
       @shards = {}
+      @shards_to_stop = []
+      @mutex = Mutex.new
     end
 
-    def run
-      @continue_running = true
-      Thread.new do
-        monitor_processes_loop
-      end
+    def do_work
+      monitor_processes
+      sleep(0.5)
     end
 
-    def halt
-      @continue_running = false
+    def after_halt
+      shards.each do |shard_no, shard|
+        if shard.alive?
+          puts "Stopping shard #{shard_no}"
+          shard.stop
+        end
+      end
     end
 
     def start_shard(shard)
-      if shards.key?(shard.shard_number)
-        raise ShardAlreadyStarted.new("Shard: #{shard.shard_number} already started!")
-      end
-
-      shards[shard.shard_number] = shard
+      @mutex.synchronize do
+        if shards.key?(shard.shard_number)
+          raise ShardAlreadyStarted.new("Shard: #{shard.shard_number} already started!")
+        end
 
-      shard.start
+        shards[shard.shard_number] = shard
+      end
     end
 
     def stop_shard(shard)
-      shards.delete(shard.shard_number)
-      shard.stop
+      @mutex.synchronize do
+        shards.delete(shard.shard_number)
+        shards_to_stop << shard
+      end
     end
 
     protected
 
-    attr_reader :shards
+    attr_reader :shards, :shards_to_stop
+
+    def monitor_processes
+      @mutex.synchronize do
+        shards_to_stop.each do |shard|
+          puts "Stopping shard #{shard.shard_number}"
+          shard.stop
+        end
+        @shards_to_stop = []
 
-    def monitor_processes_loop
-      while(@continue_running) do
         shards.each do |shard_no, shard|
           unless shard.alive?
             puts "Restarting shard #{shard_no}"
             shard.start
           end
         end
-        sleep(1)
       end
     end
 

data/lib/redis_ring/shard_config.rb

@@ -5,6 +5,10 @@ module RedisRing
     attr_reader :shard_number, :configuration
 
     def initialize(shard_number, configuration)
+      unless shard_number >= 0 && shard_number < configuration.ring_size
+        raise ArgumentError.new("shard number #{shard_number} must be between 0 and #{configuration.ring_size - 1}")
+      end
+
       @shard_number = shard_number
       @configuration = configuration
     end
@@ -74,6 +78,14 @@ module RedisRing
       file('db_files', "shard-#{shard_number}.aof")
     end
 
+    def db_mtime
+      mtime(db_file_name)
+    end
+
+    def aof_mtime
+      mtime(aof_file_name)
+    end
+
     def password
       configuration.password
     end
@@ -88,6 +100,12 @@ module RedisRing
       File.join('..', '..', *parts)
     end
 
+    def mtime(relative_path)
+      path = File.expand_path(relative_path, working_directory)
+      return nil unless File.exist?(path)
+      return File.mtime(path).to_i
+    end
+
   end
 
 end

data/lib/redis_ring/slave.rb

@@ -0,0 +1,62 @@
+module RedisRing
+
+  class Slave
+
+    attr_accessor :current_master_host, :current_master_port, :node_id
+    attr_reader :configuration, :master_rpc, :process_manager
+    attr_reader :running_shards
+
+    def initialize(configuration, master_rpc, process_manager)
+      @configuration = configuration
+      @master_rpc = master_rpc
+      @process_manager = process_manager
+      @joined = false
+      @running_shards = {}
+    end
+
+    def joined?
+      @joined
+    end
+
+    def available_shards
+      available_shards = {}
+      configuration.ring_size.times do |shard_no|
+        shard_conf = ShardConfig.new(shard_no, configuration)
+        timestamp = [shard_conf.db_mtime, shard_conf.aof_mtime].compact.max
+        available_shards[shard_no] = timestamp if timestamp
+      end
+      return available_shards
+    end
+
+    def status
+      { :joined => joined?, :running_shards => running_shards.keys, :available_shards => available_shards }
+    end
+
+    def join
+      puts "JOINING CLUSTER"
+      @joined = true
+      master_rpc.connection(current_master_host, current_master_port).node_loaded(node_id)
+    end
+
+    def start_shard(shard_number)
+      puts "STARTING SHARD #{shard_number}"
+      return if running_shards.include?(shard_number)
+      shard_conf = ShardConfig.new(shard_number, configuration)
+      shard = running_shards[shard_number] = Shard.new(shard_conf)
+      process_manager.start_shard(shard)
+    end
+
+    def stop_shard(shard_number)
+      puts "STOPPING SHARD #{shard_number}"
+      shard = running_shards[shard_number]
+      return unless shard
+      process_manager.stop_shard(shard)
+      running_shards.delete(shard_number)
+    end
+
+    def sync_shard_with(shard_number, host, port)
+    end
+
+  end
+
+end