gizzmo 0.11.3 → 0.11.4
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/gizzmo.gemspec +2 -2
- data/lib/gizzard/commands.rb +20 -1
- data/lib/gizzard/nameserver.rb +14 -7
- data/lib/gizzard/shard_template.rb +2 -0
- data/lib/gizzard/thrift.rb +2 -0
- data/lib/gizzard/transformation.rb +10 -4
- data/lib/gizzard/transformation_op.rb +42 -0
- data/lib/gizzmo.rb +14 -0
- data/test/gizzmo_spec.rb +8 -0
- metadata +4 -4
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.11.
|
1
|
+
0.11.4
|
data/gizzmo.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{gizzmo}
|
8
|
-
s.version = "0.11.
|
8
|
+
s.version = "0.11.4"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Kyle Maxwell"]
|
12
|
-
s.date = %q{2011-
|
12
|
+
s.date = %q{2011-04-19}
|
13
13
|
s.description = %q{Gizzmo is a command-line client for managing gizzard clusters.}
|
14
14
|
s.email = %q{kmaxwell@twitter.com}
|
15
15
|
s.executables = ["gizzmo", "setup_shards"]
|
data/lib/gizzard/commands.rb
CHANGED
@@ -27,6 +27,7 @@ module Gizzard
|
|
27
27
|
end
|
28
28
|
|
29
29
|
def make_manager(global_options, log)
|
30
|
+
raise "No hosts specified" unless global_options.hosts
|
30
31
|
hosts = global_options.hosts.map {|h| [h, global_options.port].join(":") }
|
31
32
|
|
32
33
|
Nameserver.new(hosts, :retries => global_options.retry,
|
@@ -502,6 +503,24 @@ module Gizzard
|
|
502
503
|
end
|
503
504
|
end
|
504
505
|
|
506
|
+
class RepairShardsCommand < Command
|
507
|
+
def run
|
508
|
+
shard_id_strings = @argv
|
509
|
+
help!("Requires at least two shard ids") unless shard_id_strings.size >= 2
|
510
|
+
shard_ids = shard_id_strings.map{|s| ShardId.parse(s)}
|
511
|
+
manager.repair_shards(shard_ids)
|
512
|
+
end
|
513
|
+
end
|
514
|
+
|
515
|
+
class DiffShardsCommand < Command
|
516
|
+
def run
|
517
|
+
shard_id_strings = @argv
|
518
|
+
help!("Requires at least two shard ids") unless shard_id_strings.size >= 2
|
519
|
+
shard_ids = shard_id_strings.map{|s| ShardId.parse(s)}
|
520
|
+
manager.diff_shards(shard_ids)
|
521
|
+
end
|
522
|
+
end
|
523
|
+
|
505
524
|
class BusyCommand < Command
|
506
525
|
def run
|
507
526
|
manager.get_busy_shards().each { |shard_info| output shard_info.to_unix }
|
@@ -793,7 +812,7 @@ module Gizzard
|
|
793
812
|
exit
|
794
813
|
end
|
795
814
|
|
796
|
-
base_name = transformations.values.
|
815
|
+
base_name = transformations.values.find {|v| v.is_a?(Hash) && !v.values.empty? }.values.find {|v| !v.nil?}.id.table_prefix.split('_').first
|
797
816
|
|
798
817
|
unless be_quiet
|
799
818
|
transformations.sort.each do |transformation, trees|
|
data/lib/gizzard/nameserver.rb
CHANGED
@@ -1,6 +1,4 @@
|
|
1
1
|
module Gizzard
|
2
|
-
Shard = Struct.new(:info, :children, :weight)
|
3
|
-
|
4
2
|
module ParallelMap
|
5
3
|
def parallel_map(enumerable, &block)
|
6
4
|
enumerable.map do |elem|
|
@@ -12,7 +10,7 @@ module Gizzard
|
|
12
10
|
end
|
13
11
|
end
|
14
12
|
|
15
|
-
class Shard
|
13
|
+
class Shard < Struct.new(:info, :children, :weight)
|
16
14
|
class << self
|
17
15
|
def canonical_table_prefix(enum, table_id = nil, base_prefix = "shard")
|
18
16
|
enum_s = "%0.4i" % enum
|
@@ -125,6 +123,16 @@ module Gizzard
|
|
125
123
|
with_retry { c.copy_shard(from_shard_id, to_shard_id) }
|
126
124
|
end
|
127
125
|
|
126
|
+
def repair_shards(*shards)
|
127
|
+
c = random_client
|
128
|
+
with_retry { c.repair_shard(*shards) }
|
129
|
+
end
|
130
|
+
|
131
|
+
def diff_shards(*shards)
|
132
|
+
c = random_client
|
133
|
+
with_retry { c.diff_shards(*shards) }
|
134
|
+
end
|
135
|
+
|
128
136
|
def respond_to?(method)
|
129
137
|
client.respond_to? method or super
|
130
138
|
end
|
@@ -166,10 +174,9 @@ module Gizzard
|
|
166
174
|
def with_retry
|
167
175
|
times ||= @retries
|
168
176
|
yield
|
169
|
-
rescue
|
170
|
-
|
171
|
-
|
172
|
-
times -= 1
|
177
|
+
rescue Exception => e
|
178
|
+
puts "retrying #{e.inspect} ..."
|
179
|
+
#times -= 1
|
173
180
|
(times < 0) ? raise : (sleep 2; retry)
|
174
181
|
end
|
175
182
|
|
data/lib/gizzard/thrift.rb
CHANGED
@@ -184,6 +184,8 @@ module Gizzard
|
|
184
184
|
|
185
185
|
thrift_method :mark_shard_busy, void, field(:id, struct(ShardId), 1), field(:busy, i32, 2), :throws => exception(GizzardException)
|
186
186
|
thrift_method :copy_shard, void, field(:source_id, struct(ShardId), 1), field(:destination_id, struct(ShardId), 2), :throws => exception(GizzardException)
|
187
|
+
thrift_method :repair_shard, void, field(:shard_ids, list(struct(ShardId)), 1), :throws => exception(GizzardException)
|
188
|
+
thrift_method :diff_shards, void, field(:shard_ids, list(struct(ShardId)), 1), :throws => exception(GizzardException)
|
187
189
|
|
188
190
|
thrift_method :dump_nameserver, list(struct(NameServerState)), field(:table_ids, list(i32), 1), :throws => exception(GizzardException)
|
189
191
|
|
@@ -12,7 +12,9 @@ module Gizzard
|
|
12
12
|
Op::CreateShard => "create_shard",
|
13
13
|
Op::AddLink => "add_link",
|
14
14
|
Op::SetForwarding => "set_forwarding",
|
15
|
-
Op::CopyShard => "copy_shard"
|
15
|
+
Op::CopyShard => "copy_shard",
|
16
|
+
Op::RepairShards => "repair_shards",
|
17
|
+
Op::DiffShards => "diff_shards"
|
16
18
|
}
|
17
19
|
|
18
20
|
OP_INVERSES = {
|
@@ -30,7 +32,9 @@ module Gizzard
|
|
30
32
|
Op::RemoveForwarding => 4,
|
31
33
|
Op::RemoveLink => 5,
|
32
34
|
Op::DeleteShard => 6,
|
33
|
-
Op::CopyShard => 7
|
35
|
+
Op::CopyShard => 7,
|
36
|
+
Op::RepairShards => 8,
|
37
|
+
Op::DiffShards => 9
|
34
38
|
}
|
35
39
|
|
36
40
|
DEFAULT_DEST_WRAPPER = 'WriteOnlyShard'
|
@@ -89,9 +93,11 @@ module Gizzard
|
|
89
93
|
|
90
94
|
prepare_inspect = op_inspect[:prepare].empty? ? "" : " PREPARE\n#{op_inspect[:prepare]}\n"
|
91
95
|
copy_inspect = op_inspect[:copy].empty? ? "" : " COPY\n#{op_inspect[:copy]}\n"
|
96
|
+
repair_inspect = op_inspect[:repair].empty? ? "" : " REPAIR\n#{op_inspect[:repair]}\n"
|
97
|
+
diff_inspect = op_inspect[:diff].empty? ? "" : " DIFF\n#{op_inspect[:diff]}\n"
|
92
98
|
cleanup_inspect = op_inspect[:cleanup].empty? ? "" : " CLEANUP\n#{op_inspect[:cleanup]}\n"
|
93
99
|
|
94
|
-
op_inspect = [prepare_inspect, copy_inspect, cleanup_inspect].join
|
100
|
+
op_inspect = [prepare_inspect, copy_inspect, repair_inspect, cleanup_inspect].join
|
95
101
|
|
96
102
|
"#{from.inspect} => #{to.inspect} :\n#{op_inspect}"
|
97
103
|
end
|
@@ -124,7 +130,7 @@ module Gizzard
|
|
124
130
|
end
|
125
131
|
|
126
132
|
def expand_jobs(jobs)
|
127
|
-
expanded = jobs.inject({:prepare => [], :copy => [], :cleanup => []}) do |ops, job|
|
133
|
+
expanded = jobs.inject({:prepare => [], :copy => [], :repair => [], :cleanup => [], :diff => []}) do |ops, job|
|
128
134
|
job_ops = job.expand(self.copy_source, involved_in_copy?(job.template), @copy_dest_wrapper)
|
129
135
|
ops.update(job_ops) {|k,a,b| a + b }
|
130
136
|
end
|
@@ -52,6 +52,48 @@ module Gizzard
|
|
52
52
|
end
|
53
53
|
end
|
54
54
|
|
55
|
+
class RepairShards < BaseOp
|
56
|
+
attr_reader :from, :to
|
57
|
+
alias template to
|
58
|
+
|
59
|
+
def initialize(*shards)
|
60
|
+
@shards = shards
|
61
|
+
end
|
62
|
+
|
63
|
+
def expand(*args); { :repair => [self] } end
|
64
|
+
|
65
|
+
def involved_shards(table_prefix, translations)
|
66
|
+
shards.map{|s| s.to_shard_id(table_prefix, translations)}
|
67
|
+
end
|
68
|
+
|
69
|
+
def apply(nameserver, table_id, base_id, table_prefix, translations)
|
70
|
+
nameserver.repair_shards(involved_shards(table_prefix, translations))
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
class DiffShards < BaseOp
|
75
|
+
attr_reader :from, :to
|
76
|
+
alias template to
|
77
|
+
|
78
|
+
def initialize(from, to)
|
79
|
+
@from = from
|
80
|
+
@to = to
|
81
|
+
end
|
82
|
+
|
83
|
+
def expand(*args); { :repair => [self] } end
|
84
|
+
|
85
|
+
def involved_shards(table_prefix, translations)
|
86
|
+
[to.to_shard_id(table_prefix, translations)]
|
87
|
+
end
|
88
|
+
|
89
|
+
def apply(nameserver, table_id, base_id, table_prefix, translations)
|
90
|
+
from_shard_id = from.to_shard_id(table_prefix, translations)
|
91
|
+
to_shard_id = to.to_shard_id(table_prefix, translations)
|
92
|
+
|
93
|
+
nameserver.diff_shards(from_shard_id, to_shard_id)
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
55
97
|
class LinkOp < BaseOp
|
56
98
|
attr_reader :from, :to
|
57
99
|
alias template to
|
data/lib/gizzmo.rb
CHANGED
@@ -24,6 +24,8 @@ DOC_STRINGS = {
|
|
24
24
|
"markbusy" => "Mark a shard as busy.",
|
25
25
|
"pair" => "Report the replica pairing structure for a list of hosts.",
|
26
26
|
"reload" => "Instruct application servers to reload the nameserver state.",
|
27
|
+
"repair-shards" => "Reconcile n shards by detecting differences and rescheduling them",
|
28
|
+
"diff-shards" => "Log differences between n shards",
|
27
29
|
"report" => "Show each unique replica structure for a given list of shards. Usually this shard list comes from << gizzmo forwardings | awk '{ print $3 }' >>.",
|
28
30
|
"setup-replica" => "Add a replica to be parallel to an existing replica, in write-only mode, ready to be copied to.",
|
29
31
|
"wrap" => "Wrapping creates a new (virtual, e.g. blocking, replicating, etc.) shard, and relinks SHARD_ID_TO_WRAP's parent links to run through the new shard.",
|
@@ -246,6 +248,18 @@ subcommands = {
|
|
246
248
|
opts.banner = "Usage: #{zero} copy SOURCE_SHARD_ID DESTINATION_SHARD_ID"
|
247
249
|
separators(opts, DOC_STRINGS["copy"])
|
248
250
|
end,
|
251
|
+
'repair-shards' => OptionParser.new do |opts|
|
252
|
+
opts.banner = "Usage: #{zero} repair-shards SHARD_IDS..."
|
253
|
+
separators(opts, DOC_STRINGS["repair-shards"])
|
254
|
+
end,
|
255
|
+
'diff-shards' => OptionParser.new do |opts|
|
256
|
+
opts.banner = "Usage: #{zero} diff-shards SHARD_IDS..."
|
257
|
+
separators(opts, DOC_STRINGS["diff-shards"])
|
258
|
+
end,
|
259
|
+
'diff-shards' => OptionParser.new do |opts|
|
260
|
+
opts.banner = "Usage: #{zero} diff-shards SOURCE_SHARD_ID DESTINATION_SHARD_ID"
|
261
|
+
separators(opts, DOC_STRINGS["diff-shards"])
|
262
|
+
end,
|
249
263
|
'busy' => OptionParser.new do |opts|
|
250
264
|
opts.banner = "Usage: #{zero} busy"
|
251
265
|
separators(opts, DOC_STRINGS["busy"])
|
data/test/gizzmo_spec.rb
CHANGED
@@ -318,6 +318,14 @@ localhost/t0_2_replicating ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int
|
|
318
318
|
it "works"
|
319
319
|
end
|
320
320
|
|
321
|
+
describe "repair-shards" do
|
322
|
+
it "works"
|
323
|
+
end
|
324
|
+
|
325
|
+
describe "diff-shards" do
|
326
|
+
it "works"
|
327
|
+
end
|
328
|
+
|
321
329
|
describe "setup-migrate" do
|
322
330
|
it "works"
|
323
331
|
end
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gizzmo
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 59
|
5
5
|
prerelease: false
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 11
|
9
|
-
-
|
10
|
-
version: 0.11.
|
9
|
+
- 4
|
10
|
+
version: 0.11.4
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Kyle Maxwell
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2011-
|
18
|
+
date: 2011-04-19 00:00:00 -07:00
|
19
19
|
default_executable:
|
20
20
|
dependencies: []
|
21
21
|
|