gizzmo 0.11.0 → 0.11.1

data/test/nameserver_spec.rb

@@ -0,0 +1,139 @@
+require File.expand_path('../spec_helper', __FILE__)
+
+describe Gizzard::Shard do
+  describe "parse_enumeration" do
+    it "parses correctly" do
+      Gizzard::Shard.parse_enumeration("edges_backwards_1_003_a").should == 3
+      Gizzard::Shard.parse_enumeration("status_0345").should == 345
+    end
+  end
+
+  describe "canonical_shard_id_map" do
+    it "returns a map of canonical to actual shard table prefixes" do
+      s = Gizzard::Shard.new(info("localhost", "t0_001_replicating", "ReplicatingShard"),
+                             [Gizzard::Shard.new(info("localhost", "shard_0001", "SqlShard"), [], 1)],
+                             1)
+      s.canonical_shard_id_map.should == {
+        id("localhost", "shard_0001_replicating") => id("localhost", "t0_001_replicating"),
+        id("localhost", "shard_0001") => id("localhost", "shard_0001")
+      }
+
+      s.canonical_shard_id_map("edges", -2).should == {
+        id("localhost", "edges_n2_0001_replicating") => id("localhost", "t0_001_replicating"),
+        id("localhost", "edges_n2_0001") => id("localhost", "shard_0001")
+      }
+
+      s.canonical_shard_id_map("groups", 0, 3).should == {
+        id("localhost", "groups_0_0003_replicating") => id("localhost", "t0_001_replicating"),
+        id("localhost", "groups_0_0003") => id("localhost", "shard_0001")
+      }
+    end
+  end
+end
+
+describe Gizzard::Nameserver do
+  before do
+    @client = Object.new
+    @second_client = Object.new
+    @nameserver = Gizzard::Nameserver.new("localhost:1234", "localhost:4567")
+    stub(@nameserver).create_client(anything) do |host|
+      { "localhost:1234" => @client, "localhost:4567" => @second_client }[host]
+    end
+  end
+
+  describe "initialize" do
+    it "takes a list of hosts and options" do
+      n = Gizzard::Nameserver.new("localhost:1234")
+      n.hosts.should == ["localhost:1234"]
+
+      n = Gizzard::Nameserver.new("localhost:1234", "localhost:4567", :dry_run => true)
+      n.hosts.should == ["localhost:1234", "localhost:4567"]
+    end
+
+    it "takes a :dry_run option that defaults to false" do
+      n = Gizzard::Nameserver.new("localhost:1234", :dry_run => true)
+      n.dryrun.should == true
+
+      n = Gizzard::Nameserver.new("localhost:1234")
+      n.dryrun.should == false
+    end
+
+    it "takes a :log option that defaults to '/tmp/gizzmo.log'" do
+      n = Gizzard::Nameserver.new("localhost:1234", :log => "/path/to/logfile")
+      n.logfile.should == "/path/to/logfile"
+
+      n = Gizzard::Nameserver.new("localhost:1234")
+      n.logfile.should == "/tmp/gizzmo.log"
+    end
+  end
+
+  describe "get_all_links" do
+    it "works..."
+  end
+
+  describe "get_all_shards" do
+    it "works..."
+  end
+
+  describe "reload_config" do
+    it "reloads config on every app server" do
+      mock(@client).reload_config
+      mock(@second_client).reload_config
+      @nameserver.reload_config
+    end
+  end
+end
+
+describe Gizzard::Nameserver::Manifest do
+  before do
+    @shardinfos = [info("localhost", "tbl_001_rep", "ReplicatingShard", "", "", 0),
+                   info("sqlhost", "tbl_001", "SqlShard", "int", "int", 0)]
+
+    @links = [link(id("localhost", "tbl_001_rep"), id("sqlhost", "tbl_001"), 1)]
+
+    @forwardings = [forwarding(0, 0, id("localhost", "tbl_001_rep"))]
+
+    @nameserver = Gizzard::Nameserver.new("localhost:1234")
+    @state = Object.new
+
+    mock(@nameserver).dump_nameserver(0) { @state }
+    mock(@state).forwardings { @forwardings }
+    mock(@state).links { @links }
+    mock(@state).shards { @shardinfos }
+  end
+
+  it "memoizes the forwardings list" do
+    @nameserver.manifest(0).forwardings.should == @forwardings
+  end
+
+  it "creates a links hash in the form of up_id => [[down_id, weight]]" do
+    @nameserver.manifest(0).links.should == {
+      id("localhost", "tbl_001_rep") => [[id("sqlhost", "tbl_001"), 1]]
+    }
+  end
+
+  it "creates a shard_infos hash in the form of shard_id => shard_info" do
+    @nameserver.manifest(0).shard_infos.should == {
+      id("localhost", "tbl_001_rep") => info("localhost", "tbl_001_rep", "ReplicatingShard", "", "", 0),
+      id("sqlhost", "tbl_001")       => info("sqlhost", "tbl_001", "SqlShard", "int", "int", 0)
+    }
+  end
+
+  it "creates a trees hash in the form of forwarding => shard tree" do
+    child  = Gizzard::Shard.new(info("sqlhost", "tbl_001", "SqlShard", "int", "int", 0), [], 1)
+    parent = Gizzard::Shard.new(info("localhost", "tbl_001_rep", "ReplicatingShard", "", "", 0), [child], 1)
+
+    @nameserver.manifest(0).trees.should == {
+      forwarding(0, 0, id("localhost", "tbl_001_rep")) => parent
+    }
+  end
+
+  it "creates a templates hash om the form of template => [forwarding]" do
+    child  = Gizzard::ShardTemplate.new("SqlShard", "sqlhost", 1, "int", "int", [])
+    parent = Gizzard::ShardTemplate.new("ReplicatingShard", "localhost", 1, "", "", [child])
+
+     @nameserver.manifest(0).templates.should == {
+      parent => [forwarding(0, 0, id("localhost", "tbl_001_rep"))]
+    }
+  end
+end

data/test/scheduler_spec.rb

@@ -0,0 +1,59 @@
+require File.expand_path('../spec_helper', __FILE__)
+
+describe Gizzard::Transformation::Scheduler do
+
+  before do
+    @nameserver = stub!.subject
+    stub(@nameserver).dryrun? { false }
+
+    @transformations = {}
+    @scheduler = Gizzard::Transformation::Scheduler.new(@nameserver, 't', @transformations)
+  end
+
+  describe "busy_shards" do
+    it "memoizes" do
+      shards = [info('127.0.0.1', 't_0_0001', 'TestShard')]
+      mock(@nameserver).get_busy_shards { shards }
+
+      @scheduler.busy_shards.should == shards.map {|s| s.id }
+      @scheduler.busy_shards.should == shards.map {|s| s.id }
+    end
+
+    it "resets after calling reload_busy_shards" do
+      shards = [info('127.0.0.1', 't_0_0001', 'TestShard')]
+      mock(@nameserver).get_busy_shards { shards }.twice
+
+      @scheduler.busy_shards.should == shards.map {|s| s.id }
+      @scheduler.reload_busy_shards
+      @scheduler.busy_shards.should == shards.map {|s| s.id }
+    end
+  end
+
+  describe "busy_hosts" do
+    it "returns a list of hosts over the threshold of copies per host" do
+      shards = []
+      stub(@nameserver).get_busy_shards { shards }
+      @scheduler = Gizzard::Transformation::Scheduler.new(@nameserver, 't', @transformations, :copies_per_host => 2)
+
+      @scheduler.busy_hosts.should == []
+
+      shards = [info('127.0.0.1', 't_0_0001', 'TestShard')]
+      @scheduler.reload_busy_shards
+      @scheduler.busy_hosts.should == []
+
+      shards = [info('127.0.0.1', 't_0_0001', 'TestShard'), info('127.0.0.1', 't_0_0002', 'TestShard')]
+      @scheduler.reload_busy_shards
+      @scheduler.busy_hosts.should == ['127.0.0.1']
+    end
+
+    it "respects passed in extra hosts" do
+      shards = []
+      stub(@nameserver).get_busy_shards { shards }
+      @scheduler = Gizzard::Transformation::Scheduler.new(@nameserver, 't', @transformations, :copies_per_host => 2)
+
+      @scheduler.busy_hosts.should == []
+      @scheduler.busy_hosts(["127.0.0.1"]).should == []
+      @scheduler.busy_hosts(["127.0.0.1", "127.0.0.1"]).should == ["127.0.0.1"]
+    end
+  end
+end

data/test/shard_template_spec.rb

@@ -0,0 +1,103 @@
+require File.expand_path('../spec_helper', __FILE__)
+
+describe Gizzard::ShardTemplate do
+  before do
+    @sql         = Gizzard::ShardTemplate.new("SqlShard", "sqlhost", 1, "", "", [])
+    @sql2        = Gizzard::ShardTemplate.new("SqlShard", "sqlhost2", 1, "", "", [])
+    @blocked     = Gizzard::ShardTemplate.new("BlockedShard", "", 1, "", "", [@sql])
+    @replicating = Gizzard::ShardTemplate.new("ReplicatingShard", "", 1, "", "", [@blocked, @sql2])
+  end
+
+  describe "concrete?" do
+    it "is false when a virtual shard type" do
+      Gizzard::Shard::VIRTUAL_SHARD_TYPES.each do |type|
+        t = Gizzard::ShardTemplate.new(type, "localhost", 1, "", "", [])
+        t.should_not be_concrete
+      end
+    end
+
+    it "is true when not a virtual shard type" do
+      @sql.should be_concrete
+    end
+  end
+
+  describe "host" do
+    it "is the template's shard if concrete" do
+      @sql.host.should == "sqlhost"
+    end
+
+    it "is the childs host if virtual and one child" do
+      @blocked.host.should == "sqlhost"
+    end
+
+    it "is the abstract host if virtual with more than one child" do
+      @replicating.host.should == Gizzard::ShardTemplate::ABSTRACT_HOST
+    end
+  end
+
+  describe "children" do
+    it "returns a sorted list" do
+      @replicating.children.should == [@blocked, @sql2].sort {|a, b| b <=> a }
+      @replicating.instance_variable_get("@children").reverse!
+      @replicating.children.should == [@blocked, @sql2].sort {|a, b| b <=> a }
+    end
+  end
+
+  describe "comparison methods" do
+    describe "shared_host?" do
+      it "is true if self and the other template share a descendant concrete identifier" do
+        other = Gizzard::ShardTemplate.new("FailingOverShard", "", 1, "", "", [@sql2])
+
+        @sql2.shared_host?(other).should        be_true
+        @replicating.shared_host?(other).should be_true
+        @blocked.shared_host?(other).should     be_false
+      end
+    end
+
+    describe "<=>" do
+      it "raises if other is not a ShardTemplate" do
+        lambda { @sql <=> "foo" }.should raise_error(ArgumentError)
+      end
+    end
+
+    describe "eql?" do
+      it "returns false if other is not a ShardTemplate" do
+        @sql.eql?("foo").should be_false
+        (@sql == "foo").should be_false
+      end
+
+      it "is structural equality" do
+        other_replicating = Marshal.load(Marshal.dump(@replicating))
+        @replicating.eql?(other_replicating).should be_true
+        (@replicating == other_replicating).should be_true
+      end
+    end
+  end
+
+  describe "config methods" do
+    describe "to_config" do
+      it "returns a human-readable string" do
+        @sql.to_config.should == "SqlShard(sqlhost,1)"
+        @blocked.to_config.should == 'BlockedShard(1) -> SqlShard(sqlhost,1)'
+        @replicating.to_config.should ==
+          'ReplicatingShard(1) -> (SqlShard(sqlhost2,1), BlockedShard(1) -> SqlShard(sqlhost,1))'
+      end
+    end
+  end
+
+  describe "config class methods" do
+    describe "parse" do
+      it "builds a shard template tree" do
+        Gizzard::ShardTemplate.parse("SqlShard(sqlhost,2)").should ==
+          Gizzard::ShardTemplate.new("SqlShard", "sqlhost", 2, "", "", [])
+
+        Gizzard::ShardTemplate.parse("SqlShard(sqlhost,1,int,int)").should ==
+          Gizzard::ShardTemplate.new("SqlShard", "sqlhost", 1, "int", "int", [])
+
+        Gizzard::ShardTemplate.parse(
+          'ReplicatingShard -> (SqlShard(sqlhost2,1), BlockedShard -> SqlShard(sqlhost,1))'
+        ).should == @replicating
+      end
+    end
+  end
+end

data/test/spec.opts

@@ -0,0 +1,7 @@
+--colour
+--format progress
+--loadby mtime
+--reverse
+--timeout 20
+--diff
+--backtrace

data/test/spec_helper.rb

@@ -0,0 +1,139 @@
+ROOT_DIR    = File.expand_path("../..", __FILE__)
+TEST_ROOT   = File.expand_path("test", ROOT_DIR)
+SERVER_ROOT = File.expand_path('test_server', TEST_ROOT)
+
+SERVER_VERSION = '0.1'
+SERVER_JAR     = File.expand_path("dist/gizzmotestserver/gizzmotestserver-#{SERVER_VERSION}.jar", SERVER_ROOT)
+
+SERVICE_PORT = 7919
+MANAGER_PORT = 7920
+JOB_PORT     = 7921
+SERVICE_DATABASE    = 'gizzard_test_integration'
+NAMESERVER_DATABASE = 'gizzard_test_integration_ns'
+
+
+require 'rubygems'
+require 'spec'
+require 'mysql'
+
+$:.unshift File.expand_path('lib', ROOT_DIR)
+require 'gizzard'
+
+Spec::Runner.configure do |c|
+  c.mock_with :rr
+end
+
+def id(h,p); Gizzard::ShardId.new(h,p) end
+def info(h,p,c,s = "",d = "",b = 0); Gizzard::ShardInfo.new(id(h,p),c,s,d,b) end
+def link(p,c,w); Gizzard::LinkInfo.new(p,c,w) end
+def forwarding(t,b,s); Gizzard::Forwarding.new(t,b,s) end
+def host(h,p,c,s = 0); Gizzard::Host.new(h,p,c,s) end
+
+def mk_template(conf_tree)
+  Gizzard::ShardTemplate.parse(conf_tree)
+end
+
+def test_server_pid
+  if pid = `ps axo pid,command`.split("\n").find {|l| l[SERVER_JAR] }
+    pid.split.first.to_i
+  end
+end
+
+def start_test_server!(manager_p = MANAGER_PORT, job_p = JOB_PORT, service_p = SERVICE_PORT)
+  unless test_server_pid
+    compile_test_server!
+
+    fork do
+      exec "cd #{SERVER_ROOT} && exec java -jar #{SERVER_JAR} #{service_p} #{job_p} #{manager_p} > /dev/null 2>&1"
+    end
+
+    sleep 3
+  end
+end
+
+def stop_test_server!
+  if pid = test_server_pid
+    Process.kill("KILL", pid)
+  end
+end
+
+def compile_test_server!
+  system "cd #{SERVER_ROOT} && sbt update package-dist" unless File.exist? SERVER_JAR
+end
+
+def mysql_connect!(host, user, pass)
+  $mysql = Mysql.new(host, user, pass)
+end
+
+def drop_database(*ds)
+  ds.each {|d| $mysql.query("drop database if exists `#{d}`") }
+end
+
+def create_database(*ds)
+  ds.each {|d| $mysql.query("create database if not exists `#{d}`") }
+end
+
+def reset_nameserver(db = NAMESERVER_DATABASE)
+  $mysql.query("delete from `#{db}`.shards")
+  $mysql.query("delete from `#{db}`.shard_children")
+  $mysql.query("delete from `#{db}`.forwardings")
+  $mysql.query("delete from `#{db}`.hosts")
+end
+
+def reset_databases!
+  drop_database SERVICE_DATABASE
+  create_database NAMESERVER_DATABASE, SERVICE_DATABASE
+
+  begin
+    reset_nameserver
+  rescue MysqlError
+
+    begin
+      m = Gizzard::Manager.new("localhost", MANAGER_PORT, '/dev/null', false)
+      m.rebuild_schema
+    rescue Errno::ECONNREFUSED
+    end
+  end
+end
+
+def read_nameserver_db(db = NAMESERVER_DATABASE)
+  { :shards      => map_rs($mysql.query("select * from `#{db}`.shards"), &method(:as_shard)),
+    :links       => map_rs($mysql.query("select * from `#{db}`.shard_children"), &method(:as_link)),
+    :forwardings => map_rs($mysql.query("select * from `#{db}`.forwardings"), &method(:as_forwarding)),
+    :hosts       => map_rs($mysql.query("select * from `#{db}`.hosts"), &method(:as_host)) }
+end
+
+def map_rs(rs)
+  a = []; rs.each_hash {|r| a << yield(r) }; a
+end
+
+def as_shard_id(h, prefix = nil)
+  attrs = ['hostname', 'table_prefix'].map {|a| prefix ? [prefix, a].join('_') : a }
+  Gizzard::ShardId.new(*h.values_at(*attrs))
+end
+
+def as_shard(h)
+  attrs = h.values_at('class_name', 'source_type', 'destination_type') << h['busy'].to_i
+  Gizzard::ShardInfo.new(as_shard_id(h), *attrs)
+end
+
+def as_link(h)
+  Gizzard::LinkInfo.new(as_shard_id(h, 'parent'), as_shard_id(h, 'child'), h['weight'].to_i)
+end
+
+def as_forwarding(h)
+  Gizzard::Forwarding.new(h['table_id'].to_i, h['base_id'].to_i, as_shard_id(h, 'shard'))
+end
+
+def as_host(h)
+  Gizzard::Host.new(h['hostname'], h['port'].to_i, h['cluster'], h['status'].to_i)
+end
+
+
+# setup
+
+mysql_connect!("localhost", '', '')
+reset_databases!
+start_test_server!
+
+at_exit { stop_test_server! }