redis_ring 0.0.2 → 0.1.0

data/spec/redis_ring/master_spec.rb

@@ -0,0 +1,174 @@
+require File.expand_path("../../spec_helper", __FILE__)
+
+describe RedisRing::Master do
+
+  describe :is_master do
+    before(:each) do
+      @master = RedisRing::Master.new(nil, nil, nil)
+    end
+
+    it "should not be master initially" do
+      @master.is_master?.should be_false
+    end
+
+    it "should be master after became_master event" do
+      @master.became_master
+
+      @master.is_master?.should be_true
+    end
+
+    it "should not be master after no_longer_is_master" do
+      @master.became_master
+      @master.no_longer_is_master
+
+      @master.is_master?.should be_false
+    end
+  end
+
+  describe :reassign_shards do
+    before(:each) do
+      @builder = ClusterBuilder.new do |b|
+        b.node("node0").port(6401)
+        b.node("node1").port(6402)
+        b.node("node2").port(6403)
+        b.node("node3").port(6404)
+        b.node("node4").port(6405)
+      end
+
+      @started_shards = []
+      @stopped_shards = []
+      @shards_per_node = Hash.new(0)
+      @stopped_on_node = Hash.new(0)
+
+      @builder.on_start_shard do |node_id, shard_no|
+        @started_shards << shard_no
+        @shards_per_node[node_id] += 1
+      end
+
+      @builder.on_stop_shard do |node_id, shard_no|
+        @stopped_shards << shard_no
+        @stopped_on_node[node_id] += 1
+      end
+
+      @master = RedisRing::Master.new(@builder.fake_connection, 16, @builder.fake_provider)
+      @master.became_master
+    end
+
+    it "should assign shards fairly to all nodes" do
+      @master.nodes_changed(@builder.node_ids)
+
+      @started_shards.sort.should == (0...16).to_a
+      @shards_per_node.values.reduce(&:+).should == 16
+      @shards_per_node.values.uniq.sort.should == [3, 4]
+    end
+
+    it "should do nothing when is not a master" do
+      @master.no_longer_is_master
+
+      @master.nodes_changed(@builder.node_ids)
+
+      @started_shards.should be_empty
+    end
+
+    it "should not reassign anything when nothing has changed" do
+      @master.nodes_changed(@builder.node_ids)
+      @started_shards = []
+      @shards_per_node = Hash.new(0)
+
+      @master.nodes_changed(@builder.node_ids)
+
+      @started_shards.should be_empty
+    end
+
+    it "should ignore not joined nodes, without assigning their portion o shards elsewhere" do
+      @builder.node("node0").joined(false)
+
+      @master.nodes_changed(@builder.node_ids)
+
+      @started_shards.size.should == 12
+      @shards_per_node["node0"].should == 0
+    end
+
+    it "should assign shards to the freshly joined node" do
+      @builder.node("node0").joined(false)
+      @master.nodes_changed(@builder.node_ids)
+      @started_shards = []
+      @shards_per_node = Hash.new(0)
+      @builder.node("node0").joined(true)
+
+      @master.node_joined("node0")
+
+      @started_shards.size.should == 4
+      @shards_per_node["node0"].should == 4
+    end
+
+    it "should not over-asign nodes if they are already running some shards" do
+      @builder.node("node0").running_shards([0, 1, 2])
+      @builder.node("node1").running_shards([3, 4, 5, 6])
+
+      @master.nodes_changed(@builder.node_ids)
+
+      @started_shards.size.should == 16 - 7
+      ((0..6).to_a - @started_shards).should == (0..6).to_a
+      @shards_per_node["node0"].should == 1
+      @shards_per_node["node1"].should == 0
+    end
+
+    it "should reassign the shards that belonged to a node that crashed" do
+      @master.nodes_changed(@builder.node_ids)
+      @started_shards = []
+      @shards_per_node = Hash.new(0)
+      @builder.nodes.delete("node0")
+
+      @master.nodes_changed(@builder.node_ids)
+
+      @started_shards.size.should == 4
+      @shards_per_node.values.uniq.should == [1]
+    end
+
+    it "should stop shards that are running on more than one node" do
+      @builder.node("node0").running_shards([0, 1, 2])
+      @builder.node("node1").running_shards([0, 1])
+      @builder.node("node2").running_shards([0, 2])
+
+      @master.nodes_changed(@builder.node_ids)
+
+      @stopped_shards.uniq.sort.should == [0, 1, 2]
+      @stopped_shards.size.should == 4
+      ([0, 1, 2] - @started_shards).should == [0, 1, 2]
+    end
+
+  end
+
+  describe :status do
+    before(:each) do
+      @builder = ClusterBuilder.new do |b|
+        b.node("node0").host("a.example.com").port(6400)
+        b.node("node1").host("b.example.com").port(6400)
+      end
+
+      @master = RedisRing::Master.new(@builder.fake_connection, 8, @builder.fake_provider)
+      @master.became_master
+      @master.nodes_changed(@builder.node_ids)
+    end
+
+    it "should return the ring_size" do
+      @master.status[:ring_size].should == 8
+    end
+
+    it "should return the shards" do
+      @master.status[:shards].should == {
+        0 => { :host => "a.example.com", :port => 6401, :status => :running },
+        1 => { :host => "a.example.com", :port => 6402, :status => :running },
+        2 => { :host => "a.example.com", :port => 6403, :status => :running },
+        3 => { :host => "a.example.com", :port => 6404, :status => :running },
+        4 => { :host => "b.example.com", :port => 6405, :status => :running },
+        5 => { :host => "b.example.com", :port => 6406, :status => :running },
+        6 => { :host => "b.example.com", :port => 6407, :status => :running },
+        7 => { :host => "b.example.com", :port => 6408, :status => :running }
+      }
+    end
+
+  end
+
+end

data/spec/redis_ring/node_spec.rb

@@ -0,0 +1,53 @@
+require File.expand_path("../../spec_helper", __FILE__)
+
+describe RedisRing::Node do
+
+  before(:each) do
+    @node = RedisRing::Node.new(@slave_rpc = FakeSlaveRPC.new.connection("some_host", 666), "some_host", 666)
+
+    @slave_rpc.status = {
+      "joined" => true,
+      "running_shards" => [1, 2, 3],
+      "available_shards" => { "1" => 1233, "2" => 4566 }
+    }
+  end
+
+  it "should return some sane defults when calling without update_status!" do
+    @node.joined?.should be_false
+    @node.running_shards.should == []
+    @node.available_shards.should == {}
+  end
+
+  it "should only update the data when update_status! is called" do
+    @node.update_status!
+
+    @node.joined?.should be_true
+    @node.running_shards.should == [1, 2, 3]
+    @node.available_shards.should == {1 => 1233, 2 => 4566}
+  end
+
+  it "should reflect start_shard in running shards without calling update_status!" do
+    @node.start_shard(9)
+
+    @node.running_shards.should == [9]
+  end
+
+  it "should send rpc to slave on start_shard" do
+    @node.start_shard(9)
+
+    @slave_rpc.started_shards.should == [9]
+  end
+
+  it "should reflect stop_shard in running shards without calling update_status!" do
+    @node.update_status!
+    @node.stop_shard(3)
+
+    @node.running_shards.should == [1, 2]
+  end
+
+  it "should send rpc to slave on start_shard" do
+    @node.stop_shard(9)
+
+    @slave_rpc.stopped_shards.should == [9]
+  end
+end

data/spec/redis_ring/slave_rpc_spec.rb

@@ -0,0 +1,46 @@
+require File.expand_path("../../spec_helper", __FILE__)
+
+describe RedisRing::SlaveRPC do
+
+  before(:each) do
+    @http_client = FakeHttpClient.new
+    @host = "example.com"
+    @port = 666
+    @rpc = RedisRing::SlaveRPC.new(@http_client).connection(@host, @port)
+  end
+
+  describe :join do
+    it "should post to joined url" do
+      @rpc.join
+
+      @http_client.sent_post?("http://example.com:666/slave/join").should be_true
+    end
+  end
+
+  describe :status do
+    it "should get and parse status" do
+      @http_client.set_response("http://example.com:666/slave/status", {"parsed" => "yes"}.to_json)
+      result = @rpc.status
+
+      @http_client.sent_get?("http://example.com:666/slave/status").should be_true
+      result.should == {"parsed" => "yes"}
+    end
+  end
+
+  describe :start_shard do
+    it "should get and parse status" do
+      @rpc.start_shard(1)
+
+      @http_client.sent_post?("http://example.com:666/slave/start_shard/1").should be_true
+    end
+  end
+
+  describe :stop_shard do
+    it "should get and parse status" do
+      @rpc.stop_shard(1)
+
+      @http_client.sent_post?("http://example.com:666/slave/stop_shard/1").should be_true
+    end
+  end
+
+end

data/spec/redis_ring/slave_spec.rb

@@ -0,0 +1,81 @@
+require File.expand_path("../../spec_helper", __FILE__)
+
+describe RedisRing::Slave do
+  before(:each) do
+    @slave = RedisRing::Slave.new(@configuration = RedisRing::Configuration.new,
+                                  @master_rpc = FakeMasterRPC.new,
+                                  @process_manager = FakeProcessManager.new)
+    @slave.node_id = "some-node"
+    @slave.current_master_host = "localhost"
+    @slave.current_master_port = 6400
+  end
+
+  it "should not be joined initially" do
+    @slave.should_not be_joined
+  end
+
+  it "should be joined after joining" do
+    @slave.join
+    @slave.should be_joined
+  end
+
+  it "should tell master to load this node on joining" do
+    @slave.join
+
+    @master_rpc.connection(@slave.current_master_host, @slave.current_master_port).nodes_loaded.should == [@slave.node_id]
+  end
+
+  it "should start shard when asked to" do
+    @slave.start_shard(0)
+
+    @process_manager.started_shards.map(&:shard_number).should == [0]
+  end
+
+  it "should not start shard if already started" do
+    @slave.start_shard(0)
+    @slave.start_shard(0)
+
+    @process_manager.started_shards.map(&:shard_number).should == [0]
+  end
+
+  it "should add shard to running when started" do
+    @slave.start_shard(0)
+    @slave.start_shard(2)
+    @slave.start_shard(4)
+
+    @slave.running_shards.keys.sort.should == [0, 2, 4]
+  end
+
+  it "should stop shard if it is running" do
+    @slave.start_shard(0)
+    @slave.stop_shard(0)
+
+    @process_manager.stopped_shards.map(&:shard_number).should == [0]
+  end
+
+  it "should do nothing if requested to stop a shard that is not running" do
+    @slave.stop_shard(0)
+
+    @process_manager.stopped_shards.should be_empty
+  end
+
+  it "it should remove stopped shard from running shards" do
+    @slave.start_shard(0)
+    @slave.start_shard(1)
+    @slave.stop_shard(0)
+
+    @slave.running_shards.keys.sort.should == [1]
+  end
+
+  it "should return status" do
+    5.times { |n| @slave.start_shard(n) }
+    @slave.join
+
+    status = @slave.status
+
+    status[:joined].should be_true
+    status[:running_shards].should == (0..4).to_a
+    status.key?(:available_shards).should be_true
+  end
+
+end

data/spec/spec_helper.rb

@@ -1,7 +1,26 @@
-$:.push File.expand_path("../lib", __FILE__)
+$:.push File.expand_path("../../lib", __FILE__)
+
+require 'rspec'
+require 'mocha'
+
+require 'simplecov'
+SimpleCov.start
 
 require 'redis_ring'
 
+def require_fake(name)
+  require File.expand_path("../fakes/fake_#{name}", __FILE__)
+end
+
+require_fake 'master_rpc'
+require_fake 'slave_rpc'
+require_fake 'process_manager'
+require_fake 'zookeeper_connection'
+require_fake 'node_provider'
+require_fake 'http_client'
+
+require File.expand_path('../cluster_builder', __FILE__)
+
 RSpec.configure do |c|
   c.color_enabled = true
   c.mock_with :mocha

data/spec/test.conf

@@ -0,0 +1,3 @@
+base_port: 6900
+base_directory: /tmp/redis-ring-spec
+ring_size: 4

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
+  - 1
   - 0
-  - 2
-  version: 0.0.2
+  version: 0.1.0
 platform: ruby
 authors: 
 - Adam Pohorecki
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-03-11 00:00:00 +01:00
+date: 2011-03-21 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -57,7 +57,7 @@ dependencies:
   type: :runtime
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  name: rspec
+  name: zookeeper
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
@@ -67,10 +67,10 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  type: :development
+  type: :runtime
   version_requirements: *id004
 - !ruby/object:Gem::Dependency 
-  name: mocha
+  name: rspec
   prerelease: false
   requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
@@ -82,6 +82,32 @@ dependencies:
         version: "0"
   type: :development
   version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: mocha
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: simplecov
+  prerelease: false
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id007
 description: RedisRing is a solution to run multiple small Redis instances intead of a single large one.
 email: 
 - adam@pohorecki.pl
@@ -103,19 +129,41 @@ files:
 - lib/monkey_patches.rb
 - lib/redis_ring.rb
 - lib/redis_ring/application.rb
+- lib/redis_ring/background_thread.rb
 - lib/redis_ring/cli.rb
 - lib/redis_ring/configuration.rb
+- lib/redis_ring/http_client.rb
+- lib/redis_ring/master.rb
+- lib/redis_ring/master_rpc.rb
+- lib/redis_ring/node.rb
 - lib/redis_ring/process_manager.rb
 - lib/redis_ring/shard.rb
 - lib/redis_ring/shard_config.rb
+- lib/redis_ring/slave.rb
+- lib/redis_ring/slave_rpc.rb
 - lib/redis_ring/version.rb
 - lib/redis_ring/web_interface.rb
+- lib/redis_ring/zookeeper_connection.rb
+- lib/redis_ring/zookeeper_observer.rb
 - redis_ring.gemspec
+- spec/cluster_builder.rb
+- spec/fakes/fake_http_client.rb
+- spec/fakes/fake_master_rpc.rb
+- spec/fakes/fake_node_provider.rb
+- spec/fakes/fake_process_manager.rb
+- spec/fakes/fake_slave_rpc.rb
+- spec/fakes/fake_zookeeper_connection.rb
 - spec/redis_ring/application_spec.rb
 - spec/redis_ring/configuration_spec.rb
+- spec/redis_ring/master_rpc_spec.rb
+- spec/redis_ring/master_spec.rb
+- spec/redis_ring/node_spec.rb
 - spec/redis_ring/shard_config_spec.rb
 - spec/redis_ring/shard_spec.rb
+- spec/redis_ring/slave_rpc_spec.rb
+- spec/redis_ring/slave_spec.rb
 - spec/spec_helper.rb
+- spec/test.conf
 has_rdoc: true
 homepage: http://github.com/psyho/redis_ring
 licenses: []