gizzmo 0.11.0 → 0.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,139 @@
1
+ require File.expand_path('../spec_helper', __FILE__)
2
+
3
+ describe Gizzard::Shard do
4
+ describe "parse_enumeration" do
5
+ it "parses correctly" do
6
+ Gizzard::Shard.parse_enumeration("edges_backwards_1_003_a").should == 3
7
+ Gizzard::Shard.parse_enumeration("status_0345").should == 345
8
+ end
9
+ end
10
+
11
+ describe "canonical_shard_id_map" do
12
+ it "returns a map of canonical to actual shard table prefixes" do
13
+ s = Gizzard::Shard.new(info("localhost", "t0_001_replicating", "ReplicatingShard"),
14
+ [Gizzard::Shard.new(info("localhost", "shard_0001", "SqlShard"), [], 1)],
15
+ 1)
16
+ s.canonical_shard_id_map.should == {
17
+ id("localhost", "shard_0001_replicating") => id("localhost", "t0_001_replicating"),
18
+ id("localhost", "shard_0001") => id("localhost", "shard_0001")
19
+ }
20
+
21
+ s.canonical_shard_id_map("edges", -2).should == {
22
+ id("localhost", "edges_n2_0001_replicating") => id("localhost", "t0_001_replicating"),
23
+ id("localhost", "edges_n2_0001") => id("localhost", "shard_0001")
24
+ }
25
+
26
+ s.canonical_shard_id_map("groups", 0, 3).should == {
27
+ id("localhost", "groups_0_0003_replicating") => id("localhost", "t0_001_replicating"),
28
+ id("localhost", "groups_0_0003") => id("localhost", "shard_0001")
29
+ }
30
+ end
31
+ end
32
+ end
33
+
34
+ describe Gizzard::Nameserver do
35
+ before do
36
+ @client = Object.new
37
+ @second_client = Object.new
38
+ @nameserver = Gizzard::Nameserver.new("localhost:1234", "localhost:4567")
39
+ stub(@nameserver).create_client(anything) do |host|
40
+ { "localhost:1234" => @client, "localhost:4567" => @second_client }[host]
41
+ end
42
+ end
43
+
44
+ describe "initialize" do
45
+ it "takes a list of hosts and options" do
46
+ n = Gizzard::Nameserver.new("localhost:1234")
47
+ n.hosts.should == ["localhost:1234"]
48
+
49
+ n = Gizzard::Nameserver.new("localhost:1234", "localhost:4567", :dry_run => true)
50
+ n.hosts.should == ["localhost:1234", "localhost:4567"]
51
+ end
52
+
53
+ it "takes a :dry_run option that defaults to false" do
54
+ n = Gizzard::Nameserver.new("localhost:1234", :dry_run => true)
55
+ n.dryrun.should == true
56
+
57
+ n = Gizzard::Nameserver.new("localhost:1234")
58
+ n.dryrun.should == false
59
+ end
60
+
61
+ it "takes a :log option that defaults to '/tmp/gizzmo.log'" do
62
+ n = Gizzard::Nameserver.new("localhost:1234", :log => "/path/to/logfile")
63
+ n.logfile.should == "/path/to/logfile"
64
+
65
+ n = Gizzard::Nameserver.new("localhost:1234")
66
+ n.logfile.should == "/tmp/gizzmo.log"
67
+ end
68
+ end
69
+
70
+ describe "get_all_links" do
71
+ it "works..."
72
+ end
73
+
74
+ describe "get_all_shards" do
75
+ it "works..."
76
+ end
77
+
78
+ describe "reload_config" do
79
+ it "reloads config on every app server" do
80
+ mock(@client).reload_config
81
+ mock(@second_client).reload_config
82
+ @nameserver.reload_config
83
+ end
84
+ end
85
+ end
86
+
87
+ describe Gizzard::Nameserver::Manifest do
88
+ before do
89
+ @shardinfos = [info("localhost", "tbl_001_rep", "ReplicatingShard", "", "", 0),
90
+ info("sqlhost", "tbl_001", "SqlShard", "int", "int", 0)]
91
+
92
+ @links = [link(id("localhost", "tbl_001_rep"), id("sqlhost", "tbl_001"), 1)]
93
+
94
+ @forwardings = [forwarding(0, 0, id("localhost", "tbl_001_rep"))]
95
+
96
+ @nameserver = Gizzard::Nameserver.new("localhost:1234")
97
+ @state = Object.new
98
+
99
+ mock(@nameserver).dump_nameserver(0) { @state }
100
+ mock(@state).forwardings { @forwardings }
101
+ mock(@state).links { @links }
102
+ mock(@state).shards { @shardinfos }
103
+ end
104
+
105
+ it "memoizes the forwardings list" do
106
+ @nameserver.manifest(0).forwardings.should == @forwardings
107
+ end
108
+
109
+ it "creates a links hash in the form of up_id => [[down_id, weight]]" do
110
+ @nameserver.manifest(0).links.should == {
111
+ id("localhost", "tbl_001_rep") => [[id("sqlhost", "tbl_001"), 1]]
112
+ }
113
+ end
114
+
115
+ it "creates a shard_infos hash in the form of shard_id => shard_info" do
116
+ @nameserver.manifest(0).shard_infos.should == {
117
+ id("localhost", "tbl_001_rep") => info("localhost", "tbl_001_rep", "ReplicatingShard", "", "", 0),
118
+ id("sqlhost", "tbl_001") => info("sqlhost", "tbl_001", "SqlShard", "int", "int", 0)
119
+ }
120
+ end
121
+
122
+ it "creates a trees hash in the form of forwarding => shard tree" do
123
+ child = Gizzard::Shard.new(info("sqlhost", "tbl_001", "SqlShard", "int", "int", 0), [], 1)
124
+ parent = Gizzard::Shard.new(info("localhost", "tbl_001_rep", "ReplicatingShard", "", "", 0), [child], 1)
125
+
126
+ @nameserver.manifest(0).trees.should == {
127
+ forwarding(0, 0, id("localhost", "tbl_001_rep")) => parent
128
+ }
129
+ end
130
+
131
+ it "creates a templates hash om the form of template => [forwarding]" do
132
+ child = Gizzard::ShardTemplate.new("SqlShard", "sqlhost", 1, "int", "int", [])
133
+ parent = Gizzard::ShardTemplate.new("ReplicatingShard", "localhost", 1, "", "", [child])
134
+
135
+ @nameserver.manifest(0).templates.should == {
136
+ parent => [forwarding(0, 0, id("localhost", "tbl_001_rep"))]
137
+ }
138
+ end
139
+ end
@@ -0,0 +1,59 @@
1
+ require File.expand_path('../spec_helper', __FILE__)
2
+
3
+ describe Gizzard::Transformation::Scheduler do
4
+
5
+ before do
6
+ @nameserver = stub!.subject
7
+ stub(@nameserver).dryrun? { false }
8
+
9
+ @transformations = {}
10
+ @scheduler = Gizzard::Transformation::Scheduler.new(@nameserver, 't', @transformations)
11
+ end
12
+
13
+ describe "busy_shards" do
14
+ it "memoizes" do
15
+ shards = [info('127.0.0.1', 't_0_0001', 'TestShard')]
16
+ mock(@nameserver).get_busy_shards { shards }
17
+
18
+ @scheduler.busy_shards.should == shards.map {|s| s.id }
19
+ @scheduler.busy_shards.should == shards.map {|s| s.id }
20
+ end
21
+
22
+ it "resets after calling reload_busy_shards" do
23
+ shards = [info('127.0.0.1', 't_0_0001', 'TestShard')]
24
+ mock(@nameserver).get_busy_shards { shards }.twice
25
+
26
+ @scheduler.busy_shards.should == shards.map {|s| s.id }
27
+ @scheduler.reload_busy_shards
28
+ @scheduler.busy_shards.should == shards.map {|s| s.id }
29
+ end
30
+ end
31
+
32
+ describe "busy_hosts" do
33
+ it "returns a list of hosts over the threshold of copies per host" do
34
+ shards = []
35
+ stub(@nameserver).get_busy_shards { shards }
36
+ @scheduler = Gizzard::Transformation::Scheduler.new(@nameserver, 't', @transformations, :copies_per_host => 2)
37
+
38
+ @scheduler.busy_hosts.should == []
39
+
40
+ shards = [info('127.0.0.1', 't_0_0001', 'TestShard')]
41
+ @scheduler.reload_busy_shards
42
+ @scheduler.busy_hosts.should == []
43
+
44
+ shards = [info('127.0.0.1', 't_0_0001', 'TestShard'), info('127.0.0.1', 't_0_0002', 'TestShard')]
45
+ @scheduler.reload_busy_shards
46
+ @scheduler.busy_hosts.should == ['127.0.0.1']
47
+ end
48
+
49
+ it "respects passed in extra hosts" do
50
+ shards = []
51
+ stub(@nameserver).get_busy_shards { shards }
52
+ @scheduler = Gizzard::Transformation::Scheduler.new(@nameserver, 't', @transformations, :copies_per_host => 2)
53
+
54
+ @scheduler.busy_hosts.should == []
55
+ @scheduler.busy_hosts(["127.0.0.1"]).should == []
56
+ @scheduler.busy_hosts(["127.0.0.1", "127.0.0.1"]).should == ["127.0.0.1"]
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,103 @@
1
+ require File.expand_path('../spec_helper', __FILE__)
2
+
3
+ describe Gizzard::ShardTemplate do
4
+ before do
5
+ @sql = Gizzard::ShardTemplate.new("SqlShard", "sqlhost", 1, "", "", [])
6
+ @sql2 = Gizzard::ShardTemplate.new("SqlShard", "sqlhost2", 1, "", "", [])
7
+ @blocked = Gizzard::ShardTemplate.new("BlockedShard", "", 1, "", "", [@sql])
8
+ @replicating = Gizzard::ShardTemplate.new("ReplicatingShard", "", 1, "", "", [@blocked, @sql2])
9
+ end
10
+
11
+ describe "concrete?" do
12
+ it "is false when a virtual shard type" do
13
+ Gizzard::Shard::VIRTUAL_SHARD_TYPES.each do |type|
14
+ t = Gizzard::ShardTemplate.new(type, "localhost", 1, "", "", [])
15
+ t.should_not be_concrete
16
+ end
17
+ end
18
+
19
+ it "is true when not a virtual shard type" do
20
+ @sql.should be_concrete
21
+ end
22
+ end
23
+
24
+ describe "host" do
25
+ it "is the template's shard if concrete" do
26
+ @sql.host.should == "sqlhost"
27
+ end
28
+
29
+ it "is the childs host if virtual and one child" do
30
+ @blocked.host.should == "sqlhost"
31
+ end
32
+
33
+ it "is the abstract host if virtual with more than one child" do
34
+ @replicating.host.should == Gizzard::ShardTemplate::ABSTRACT_HOST
35
+ end
36
+ end
37
+
38
+ describe "children" do
39
+ it "returns a sorted list" do
40
+ @replicating.children.should == [@blocked, @sql2].sort {|a, b| b <=> a }
41
+ @replicating.instance_variable_get("@children").reverse!
42
+ @replicating.children.should == [@blocked, @sql2].sort {|a, b| b <=> a }
43
+ end
44
+ end
45
+
46
+ describe "comparison methods" do
47
+ describe "shared_host?" do
48
+ it "is true if self and the other template share a descendant concrete identifier" do
49
+ other = Gizzard::ShardTemplate.new("FailingOverShard", "", 1, "", "", [@sql2])
50
+
51
+ @sql2.shared_host?(other).should be_true
52
+ @replicating.shared_host?(other).should be_true
53
+ @blocked.shared_host?(other).should be_false
54
+ end
55
+ end
56
+
57
+ describe "<=>" do
58
+ it "raises if other is not a ShardTemplate" do
59
+ lambda { @sql <=> "foo" }.should raise_error(ArgumentError)
60
+ end
61
+ end
62
+
63
+ describe "eql?" do
64
+ it "returns false if other is not a ShardTemplate" do
65
+ @sql.eql?("foo").should be_false
66
+ (@sql == "foo").should be_false
67
+ end
68
+
69
+ it "is structural equality" do
70
+ other_replicating = Marshal.load(Marshal.dump(@replicating))
71
+ @replicating.eql?(other_replicating).should be_true
72
+ (@replicating == other_replicating).should be_true
73
+ end
74
+ end
75
+ end
76
+
77
+ describe "config methods" do
78
+ describe "to_config" do
79
+ it "returns a human-readable string" do
80
+ @sql.to_config.should == "SqlShard(sqlhost,1)"
81
+ @blocked.to_config.should == 'BlockedShard(1) -> SqlShard(sqlhost,1)'
82
+ @replicating.to_config.should ==
83
+ 'ReplicatingShard(1) -> (SqlShard(sqlhost2,1), BlockedShard(1) -> SqlShard(sqlhost,1))'
84
+ end
85
+ end
86
+ end
87
+
88
+ describe "config class methods" do
89
+ describe "parse" do
90
+ it "builds a shard template tree" do
91
+ Gizzard::ShardTemplate.parse("SqlShard(sqlhost,2)").should ==
92
+ Gizzard::ShardTemplate.new("SqlShard", "sqlhost", 2, "", "", [])
93
+
94
+ Gizzard::ShardTemplate.parse("SqlShard(sqlhost,1,int,int)").should ==
95
+ Gizzard::ShardTemplate.new("SqlShard", "sqlhost", 1, "int", "int", [])
96
+
97
+ Gizzard::ShardTemplate.parse(
98
+ 'ReplicatingShard -> (SqlShard(sqlhost2,1), BlockedShard -> SqlShard(sqlhost,1))'
99
+ ).should == @replicating
100
+ end
101
+ end
102
+ end
103
+ end
data/test/spec.opts ADDED
@@ -0,0 +1,7 @@
1
+ --colour
2
+ --format progress
3
+ --loadby mtime
4
+ --reverse
5
+ --timeout 20
6
+ --diff
7
+ --backtrace
@@ -0,0 +1,139 @@
1
+ ROOT_DIR = File.expand_path("../..", __FILE__)
2
+ TEST_ROOT = File.expand_path("test", ROOT_DIR)
3
+ SERVER_ROOT = File.expand_path('test_server', TEST_ROOT)
4
+
5
+ SERVER_VERSION = '0.1'
6
+ SERVER_JAR = File.expand_path("dist/gizzmotestserver/gizzmotestserver-#{SERVER_VERSION}.jar", SERVER_ROOT)
7
+
8
+ SERVICE_PORT = 7919
9
+ MANAGER_PORT = 7920
10
+ JOB_PORT = 7921
11
+ SERVICE_DATABASE = 'gizzard_test_integration'
12
+ NAMESERVER_DATABASE = 'gizzard_test_integration_ns'
13
+
14
+
15
+ require 'rubygems'
16
+ require 'spec'
17
+ require 'mysql'
18
+
19
+ $:.unshift File.expand_path('lib', ROOT_DIR)
20
+ require 'gizzard'
21
+
22
+ Spec::Runner.configure do |c|
23
+ c.mock_with :rr
24
+ end
25
+
26
+ def id(h,p); Gizzard::ShardId.new(h,p) end
27
+ def info(h,p,c,s = "",d = "",b = 0); Gizzard::ShardInfo.new(id(h,p),c,s,d,b) end
28
+ def link(p,c,w); Gizzard::LinkInfo.new(p,c,w) end
29
+ def forwarding(t,b,s); Gizzard::Forwarding.new(t,b,s) end
30
+ def host(h,p,c,s = 0); Gizzard::Host.new(h,p,c,s) end
31
+
32
+ def mk_template(conf_tree)
33
+ Gizzard::ShardTemplate.parse(conf_tree)
34
+ end
35
+
36
+ def test_server_pid
37
+ if pid = `ps axo pid,command`.split("\n").find {|l| l[SERVER_JAR] }
38
+ pid.split.first.to_i
39
+ end
40
+ end
41
+
42
+ def start_test_server!(manager_p = MANAGER_PORT, job_p = JOB_PORT, service_p = SERVICE_PORT)
43
+ unless test_server_pid
44
+ compile_test_server!
45
+
46
+ fork do
47
+ exec "cd #{SERVER_ROOT} && exec java -jar #{SERVER_JAR} #{service_p} #{job_p} #{manager_p} > /dev/null 2>&1"
48
+ end
49
+
50
+ sleep 3
51
+ end
52
+ end
53
+
54
+ def stop_test_server!
55
+ if pid = test_server_pid
56
+ Process.kill("KILL", pid)
57
+ end
58
+ end
59
+
60
+ def compile_test_server!
61
+ system "cd #{SERVER_ROOT} && sbt update package-dist" unless File.exist? SERVER_JAR
62
+ end
63
+
64
+ def mysql_connect!(host, user, pass)
65
+ $mysql = Mysql.new(host, user, pass)
66
+ end
67
+
68
+ def drop_database(*ds)
69
+ ds.each {|d| $mysql.query("drop database if exists `#{d}`") }
70
+ end
71
+
72
+ def create_database(*ds)
73
+ ds.each {|d| $mysql.query("create database if not exists `#{d}`") }
74
+ end
75
+
76
+ def reset_nameserver(db = NAMESERVER_DATABASE)
77
+ $mysql.query("delete from `#{db}`.shards")
78
+ $mysql.query("delete from `#{db}`.shard_children")
79
+ $mysql.query("delete from `#{db}`.forwardings")
80
+ $mysql.query("delete from `#{db}`.hosts")
81
+ end
82
+
83
+ def reset_databases!
84
+ drop_database SERVICE_DATABASE
85
+ create_database NAMESERVER_DATABASE, SERVICE_DATABASE
86
+
87
+ begin
88
+ reset_nameserver
89
+ rescue MysqlError
90
+
91
+ begin
92
+ m = Gizzard::Manager.new("localhost", MANAGER_PORT, '/dev/null', false)
93
+ m.rebuild_schema
94
+ rescue Errno::ECONNREFUSED
95
+ end
96
+ end
97
+ end
98
+
99
+ def read_nameserver_db(db = NAMESERVER_DATABASE)
100
+ { :shards => map_rs($mysql.query("select * from `#{db}`.shards"), &method(:as_shard)),
101
+ :links => map_rs($mysql.query("select * from `#{db}`.shard_children"), &method(:as_link)),
102
+ :forwardings => map_rs($mysql.query("select * from `#{db}`.forwardings"), &method(:as_forwarding)),
103
+ :hosts => map_rs($mysql.query("select * from `#{db}`.hosts"), &method(:as_host)) }
104
+ end
105
+
106
+ def map_rs(rs)
107
+ a = []; rs.each_hash {|r| a << yield(r) }; a
108
+ end
109
+
110
+ def as_shard_id(h, prefix = nil)
111
+ attrs = ['hostname', 'table_prefix'].map {|a| prefix ? [prefix, a].join('_') : a }
112
+ Gizzard::ShardId.new(*h.values_at(*attrs))
113
+ end
114
+
115
+ def as_shard(h)
116
+ attrs = h.values_at('class_name', 'source_type', 'destination_type') << h['busy'].to_i
117
+ Gizzard::ShardInfo.new(as_shard_id(h), *attrs)
118
+ end
119
+
120
+ def as_link(h)
121
+ Gizzard::LinkInfo.new(as_shard_id(h, 'parent'), as_shard_id(h, 'child'), h['weight'].to_i)
122
+ end
123
+
124
+ def as_forwarding(h)
125
+ Gizzard::Forwarding.new(h['table_id'].to_i, h['base_id'].to_i, as_shard_id(h, 'shard'))
126
+ end
127
+
128
+ def as_host(h)
129
+ Gizzard::Host.new(h['hostname'], h['port'].to_i, h['cluster'], h['status'].to_i)
130
+ end
131
+
132
+
133
+ # setup
134
+
135
+ mysql_connect!("localhost", '', '')
136
+ reset_databases!
137
+ start_test_server!
138
+
139
+ at_exit { stop_test_server! }