gizzmo 0.11.0 → 0.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/Rakefile CHANGED
@@ -1,3 +1,4 @@
1
+ ROOT_DIR = File.expand_path(File.dirname(__FILE__))
1
2
  require 'rubygems'
2
3
  require 'rake'
3
4
 
@@ -16,28 +17,22 @@ rescue LoadError
16
17
  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
17
18
  end
18
19
 
19
- require 'rake/testtask'
20
- Rake::TestTask.new(:test) do |test|
21
- test.libs << 'lib' << 'test'
22
- test.pattern = 'test/**/test_*.rb'
23
- test.verbose = true
24
- end
25
20
 
26
21
  begin
27
- require 'rcov/rcovtask'
28
- Rcov::RcovTask.new do |test|
29
- test.libs << 'test'
30
- test.pattern = 'test/**/test_*.rb'
31
- test.verbose = true
22
+ require 'spec/rake/spectask'
23
+ Spec::Rake::SpecTask.new(:spec) do |t|
24
+ spec_opts = File.expand_path('test/spec.opts', ROOT_DIR)
25
+ if File.exist? spec_opts
26
+ t.spec_opts = ['--options', "\"#{spec_opts}\""]
27
+ end
28
+ t.spec_files = FileList['test/**/*_spec.rb']
32
29
  end
33
30
  rescue LoadError
34
- task :rcov do
35
- abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
36
- end
31
+ $stderr.puts "RSpec required to run tests."
37
32
  end
38
33
 
39
34
  task :test do
40
- puts
35
+ puts
41
36
  puts "=" * 79
42
37
  puts "You might want to read the README before running tests."
43
38
  puts "=" * 79
@@ -45,7 +40,7 @@ task :test do
45
40
  exec File.join(File.dirname(__FILE__), "test", "test.sh")
46
41
  end
47
42
 
48
- task :default => :test
43
+ task :default => :spec
49
44
 
50
45
  require 'rake/rdoctask'
51
46
  Rake::RDocTask.new do |rdoc|
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.11.0
1
+ 0.11.1
data/bin/setup_shards ADDED
@@ -0,0 +1,173 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'yaml'
4
+ require 'ostruct'
5
+ require 'optparse'
6
+ require 'gizzard'
7
+
8
+ require 'gizzard/nameserver'
9
+ require 'gizzard/migrator'
10
+ require 'gizzard/transformation'
11
+ require 'gizzard/shard_template'
12
+
13
+ def usage(parser, reason)
14
+ puts
15
+ puts "ERROR: #{reason}"
16
+ puts
17
+ puts parser
18
+ puts
19
+ exit 1
20
+ end
21
+
22
+ options = {
23
+ :config_filename => ENV['HOME'] + "/.topology.yml",
24
+ :dry_run => false,
25
+ :max_copies => 20,
26
+ :table_id => nil
27
+ }
28
+
29
+ parser = OptionParser.new do |opts|
30
+ opts.banner = "Usage: #{$0} [options]"
31
+ opts.separator "Example: #{$0} -f topology.yml"
32
+
33
+ opts.on("-f", "--config=FILENAME", "load database topology config (default: #{options[:config_filename]})") do |filename|
34
+ options[:config_filename] = filename
35
+ end
36
+ opts.on("-i", "--id=TABLE_ID", "table id to use (default: #{options[:table_id]})") do |table_id|
37
+ options[:table_id] = table_id.to_i
38
+ end
39
+ opts.on("-s", "--shards=N", "create N shards") do |n|
40
+ options[:total_shards] = n.to_i
41
+ end
42
+ opts.on("-t", "--table=TABLE_NAME", "table name") do |table_name|
43
+ options[:table_name] = table_name
44
+ end
45
+ opts.on("-c", "--class=CLASS_NAME", "shard class to create") do |class_name|
46
+ options[:shard_class] = class_name
47
+ end
48
+ opts.on("-n", "--dry-run", "don't actually send gizzard commands") do
49
+ options[:dry_run] = true
50
+ end
51
+ opts.on("-x", "--max=COPIES", "max concurrent copies (default: #{options[:max_copies]})") do |n|
52
+ options[:max_copies] = n.to_i
53
+ end
54
+ end
55
+
56
+ parser.parse!(ARGV)
57
+
58
+ config = begin
59
+ YAML.load_file(options[:config_filename])
60
+ rescue => e
61
+ puts "Exception while reading config file: #{e}"
62
+ {}
63
+ end
64
+
65
+ total_shards = options[:total_shards] || config['total_shards']
66
+ usage(parser, "Must define total_shards or -s") unless total_shards
67
+
68
+ table_name = options[:table_name] || config['table_name']
69
+ usage(parser, "Must define table_name or -t") unless table_name
70
+
71
+ migrator_config = Gizzard::MigratorConfig.new
72
+ migrator_config.prefix = table_name
73
+ migrator_config.table_id = options[:table_id]
74
+ migrator_config.source_type = config['source_type'] || ''
75
+ migrator_config.destination_type = config['destination_type'] || ''
76
+ migrator_config.forwarding_space = config['forwarding_space'] || 2 ** 64
77
+ migrator_config.forwarding_space_min = config['forwarding_space_min'] || 2 ** 63 * -1
78
+
79
+ querying_nameserver = Gizzard::Nameserver.new(config['app_hosts'] || 'localhost', :dry_run => false)
80
+ mutating_nameserver = Gizzard::Nameserver.new(config['app_hosts'] || 'localhost', :dry_run => options[:dry_run])
81
+
82
+ usage(parser, "Config file must contain a 'partitions' definition") unless config['partitions']
83
+
84
+ # Where the magic happens:
85
+
86
+ new_templates = config['partitions'].map { |p| Gizzard::ShardTemplate.from_config(migrator_config, p) }
87
+
88
+ EXISTING_CONFIG_CACHE = File.expand_path(".existing_shards.marshal")
89
+
90
+ if File.exist? EXISTING_CONFIG_CACHE
91
+ puts "Loading cache from previous operation. If this is not desired, delete the cache file at #{EXISTING_CONFIG_CACHE}"
92
+ manifest = Marshal.load(File.read(EXISTING_CONFIG_CACHE))
93
+ raise "error loading cache!" unless manifest.is_a? Gizzard::Manifest
94
+ else
95
+ puts "Querying nameserver..."
96
+ manifest = Gizzard::Manifest.new(querying_nameserver, migrator_config)
97
+ File.open(EXISTING_CONFIG_CACHE, 'w') { |f| f.write Marshal.dump(manifest) }
98
+ end
99
+
100
+ migrator_config.manifest = manifest
101
+ existing_map = manifest.template_map
102
+
103
+
104
+ puts "\nCalculating changes...\n"
105
+
106
+
107
+ migrator = Gizzard::Migrator.new(existing_map, new_templates, total_shards, migrator_config)
108
+
109
+ puts "\nUNCHANGED:"
110
+ pp migrator.unchanged_templates
111
+
112
+ puts "\nSIMILAR:"
113
+ pp migrator.similar_templates
114
+
115
+ puts "\nNEW:"
116
+ pp migrator.new_templates
117
+
118
+ puts "\nUNRECOGNIZED:"
119
+ pp migrator.unrecognized_templates
120
+
121
+ unless (transformations = migrator.transformations).empty?
122
+ puts "\nTRANSFORMATIONS:"
123
+ transformations.each { |t| puts ""; p t }
124
+
125
+ printf "Apply migration? [yN] "; $stdout.flush
126
+
127
+ if $stdin.gets.chomp == 'y'
128
+ pages = transformations.inject([]) { |pages, t| pages.concat t.paginate(options[:max_copies]) }
129
+
130
+ pages.each_with_index do |page, page_idx|
131
+ puts "\nTRANSFORMATION #{page_idx + 1} / #{pages.length}:"
132
+
133
+ puts page.inspect(true)
134
+
135
+ t_start = Time.now
136
+
137
+ if page.must_copy?
138
+ puts " Preparing nameserver for copies."
139
+ page.prepare! mutating_nameserver, migrator_config
140
+ mutating_nameserver.reload_forwardings
141
+
142
+ puts " Scheduling copies."
143
+ page.copy! mutating_nameserver, migrator_config
144
+
145
+ puts " Waiting for copies to finish."
146
+ page.wait_for_copies mutating_nameserver, migrator_config
147
+
148
+ puts " Finalizing nameserver changes."
149
+ page.cleanup! mutating_nameserver, migrator_config
150
+ mutating_nameserver.reload_forwardings
151
+ else
152
+ puts " Applying nameserver changes."
153
+ page.apply! mutating_nameserver, migrator_config
154
+ end
155
+
156
+ puts "--- Time Elapsed: %0.3f ---" % (Time.now - t_start).to_f
157
+ end
158
+
159
+ mutating_nameserver.reload_forwardings
160
+
161
+ puts "Done!"
162
+
163
+ else
164
+ puts "Aborting migration."
165
+ end
166
+
167
+ else
168
+ puts "No migration needed."
169
+ end
170
+
171
+ # remove the existing config state since we're done, and the system is
172
+ # in a known good state.
173
+ File.unlink EXISTING_CONFIG_CACHE
data/lib/gizzard.rb CHANGED
@@ -2,4 +2,8 @@ $: << File.dirname(__FILE__)
2
2
  module Gizzard; end
3
3
  require "gizzard/thrift"
4
4
  require "gizzard/commands"
5
+ require "gizzard/nameserver"
6
+ require "gizzard/shard_template"
7
+ require "gizzard/migrator"
8
+ require "gizzard/transformation"
5
9
  require "gizzard/digest"
@@ -1,29 +1,51 @@
1
1
  require "pp"
2
+ require "set"
2
3
  require "digest/md5"
3
4
 
4
5
  module Gizzard
5
6
  class Command
6
- include Thrift
7
7
 
8
8
  attr_reader :buffer
9
9
 
10
- def self.run(command_name, global_options, argv, subcommand_options, log, service=nil)
11
- command_class = Gizzard.const_get("#{classify(command_name)}Command")
12
- service = command_class.make_service(global_options, log) if service.nil?
13
- command = command_class.new(service, global_options, argv, subcommand_options)
14
- command.run
15
- if command.buffer && command_name = global_options.render.shift
16
- run(command_name, global_options, command.buffer, OpenStruct.new, log, service)
10
+ class << self
11
+ def run(command_name, global_options, argv, subcommand_options, log)
12
+ command_class = Gizzard.const_get("#{classify(command_name)}Command")
13
+
14
+ @manager ||= make_manager(global_options, log)
15
+ @job_injector ||= make_job_injector(global_options, log)
16
+
17
+ command = command_class.new(@manager, @job_injector, global_options, argv, subcommand_options)
18
+ command.run
19
+
20
+ if command.buffer && command_name = global_options.render.shift
21
+ run(command_name, global_options, command.buffer, OpenStruct.new, log)
22
+ end
23
+ end
24
+
25
+ def classify(string)
26
+ string.split(/\W+/).map{|s| s.capitalize }.join("")
27
+ end
28
+
29
+ def make_manager(global_options, log)
30
+ hosts = global_options.hosts.map {|h| [h, global_options.port].join(":") }
31
+
32
+ Nameserver.new(hosts, :retries => global_options.retry,
33
+ :log => log,
34
+ :framed => global_options.framed,
35
+ :dry_run => global_options.dry)
17
36
  end
18
- end
19
37
 
20
- def self.classify(string)
21
- string.split(/\W+/).map { |s| s.capitalize }.join("")
38
+ def make_job_injector(global_options, log)
39
+ RetryProxy.new global_options.retry,
40
+ JobInjector.new(global_options.hosts.first, global_options.injector_port, log, true, global_options.dry)
41
+ end
22
42
  end
23
43
 
24
- attr_reader :service, :global_options, :argv, :command_options
25
- def initialize(service, global_options, argv, command_options)
26
- @service = service
44
+ attr_reader :manager, :job_injector, :global_options, :argv, :command_options
45
+
46
+ def initialize(manager, job_injector, global_options, argv, command_options)
47
+ @manager = manager
48
+ @job_injector = job_injector
27
49
  @global_options = global_options
28
50
  @argv = argv
29
51
  @command_options = command_options
@@ -62,49 +84,35 @@ module Gizzard
62
84
  end
63
85
  end
64
86
 
65
- class ShardCommand < Command
66
- def self.make_service(global_options, log)
67
- RetryProxy.new global_options.retry.to_i,
68
- Gizzard::Thrift::ShardManager.new(global_options.host, global_options.port, log, global_options.framed, global_options.dry)
69
- end
70
- end
71
-
72
- class JobCommand < Command
73
- def self.make_service(global_options, log)
74
- RetryProxy.new global_options.retry.to_i ,
75
- Gizzard::Thrift::JobManager.new(global_options.host, global_options.port + 2, log, global_options.framed, global_options.dry)
76
- end
77
- end
78
-
79
- class AddforwardingCommand < ShardCommand
87
+ class AddforwardingCommand < Command
80
88
  def run
81
89
  help! if argv.length != 3
82
90
  table_id, base_id, shard_id_text = argv
83
91
  shard_id = ShardId.parse(shard_id_text)
84
- service.set_forwarding(Forwarding.new(table_id.to_i, base_id.to_i, shard_id))
92
+ manager.set_forwarding(Forwarding.new(table_id.to_i, base_id.to_i, shard_id))
85
93
  end
86
94
  end
87
95
 
88
- class DeleteforwardingCommand < ShardCommand
96
+ class DeleteforwardingCommand < Command
89
97
  def run
90
98
  help! if argv.length != 3
91
99
  table_id, base_id, shard_id_text = argv
92
100
  shard_id = ShardId.parse(shard_id_text)
93
- service.remove_forwarding(Forwarding.new(table_id.to_i, base_id.to_i, shard_id))
101
+ manager.remove_forwarding(Forwarding.new(table_id.to_i, base_id.to_i, shard_id))
94
102
  end
95
103
  end
96
104
 
97
- class HostsCommand < ShardCommand
105
+ class HostsCommand < Command
98
106
  def run
99
- service.list_hostnames.map do |host|
107
+ manager.list_hostnames.map do |host|
100
108
  puts host
101
109
  end
102
110
  end
103
111
  end
104
112
 
105
- class ForwardingsCommand < ShardCommand
113
+ class ForwardingsCommand < Command
106
114
  def run
107
- service.get_forwardings().sort_by do |f|
115
+ manager.get_forwardings.sort_by do |f|
108
116
  [ ((f.table_id.abs << 1) + (f.table_id < 0 ? 1 : 0)), f.base_id ]
109
117
  end.reject do |forwarding|
110
118
  @command_options.table_ids && !@command_options.table_ids.include?(forwarding.table_id)
@@ -114,7 +122,7 @@ module Gizzard
114
122
  end
115
123
  end
116
124
 
117
- class SubtreeCommand < ShardCommand
125
+ class SubtreeCommand < Command
118
126
  def run
119
127
  @roots = []
120
128
  argv.each do |arg|
@@ -128,7 +136,7 @@ module Gizzard
128
136
  end
129
137
 
130
138
  def roots_of(id)
131
- links = service.list_upward_links(id)
139
+ links = manager.list_upward_links(id)
132
140
  if links.empty?
133
141
  [id]
134
142
  else
@@ -137,7 +145,7 @@ module Gizzard
137
145
  end
138
146
 
139
147
  def down(id, depth = 0)
140
- service.list_downward_links(id).map do |link|
148
+ manager.list_downward_links(id).map do |link|
141
149
  printable = " " * depth + link.down_id.to_unix
142
150
  output printable
143
151
  down(link.down_id, depth + 1)
@@ -145,21 +153,10 @@ module Gizzard
145
153
  end
146
154
  end
147
155
 
148
- class ReloadCommand < ShardCommand
156
+ class ReloadCommand < Command
149
157
  def run
150
158
  if global_options.force || ask
151
- if @argv
152
- # allow hosts to be given on the command line
153
- @argv.each do |hostname|
154
- output hostname
155
- opts = global_options.dup
156
- opts.host = hostname
157
- s = self.class.make_service(opts, global_options.log || "./gizzmo.log")
158
- s.reload_forwardings
159
- end
160
- else
161
- service.reload_forwardings
162
- end
159
+ manager.reload_config
163
160
  else
164
161
  STDERR.puts "aborted"
165
162
  end
@@ -171,17 +168,17 @@ module Gizzard
171
168
  end
172
169
  end
173
170
 
174
- class DeleteCommand < ShardCommand
171
+ class DeleteCommand < Command
175
172
  def run
176
173
  argv.each do |arg|
177
174
  id = ShardId.parse(arg)
178
- service.delete_shard(id)
175
+ manager.delete_shard(id)
179
176
  output id.to_unix
180
177
  end
181
178
  end
182
179
  end
183
180
 
184
- class AddlinkCommand < ShardCommand
181
+ class AddlinkCommand < Command
185
182
  def run
186
183
  up_id, down_id, weight = argv
187
184
  help! if argv.length != 3
@@ -189,29 +186,29 @@ module Gizzard
189
186
  up_id = ShardId.parse(up_id)
190
187
  down_id = ShardId.parse(down_id)
191
188
  link = LinkInfo.new(up_id, down_id, weight)
192
- service.add_link(link.up_id, link.down_id, link.weight)
189
+ manager.add_link(link.up_id, link.down_id, link.weight)
193
190
  output link.to_unix
194
191
  end
195
192
  end
196
193
 
197
- class UnlinkCommand < ShardCommand
194
+ class UnlinkCommand < Command
198
195
  def run
199
196
  up_id, down_id = argv
200
197
  up_id = ShardId.parse(up_id)
201
198
  down_id = ShardId.parse(down_id)
202
- service.remove_link(up_id, down_id)
199
+ manager.remove_link(up_id, down_id)
203
200
  end
204
201
  end
205
202
 
206
- class UnwrapCommand < ShardCommand
203
+ class UnwrapCommand < Command
207
204
  def run
208
205
  shard_ids = argv
209
206
  help! "No shards specified" if shard_ids.empty?
210
207
  shard_ids.each do |shard_id_string|
211
208
  shard_id = ShardId.parse(shard_id_string)
212
209
 
213
- upward_links = service.list_upward_links(shard_id)
214
- downward_links = service.list_downward_links(shard_id)
210
+ upward_links = manager.list_upward_links(shard_id)
211
+ downward_links = manager.list_downward_links(shard_id)
215
212
 
216
213
  if upward_links.length == 0 or downward_links.length == 0
217
214
  STDERR.puts "Shard #{shard_id_string} must not be a root or leaf"
@@ -220,19 +217,19 @@ module Gizzard
220
217
 
221
218
  upward_links.each do |uplink|
222
219
  downward_links.each do |downlink|
223
- service.add_link(uplink.up_id, downlink.down_id, uplink.weight)
220
+ manager.add_link(uplink.up_id, downlink.down_id, uplink.weight)
224
221
  new_link = LinkInfo.new(uplink.up_id, downlink.down_id, uplink.weight)
225
- service.remove_link(uplink.up_id, uplink.down_id)
226
- service.remove_link(downlink.up_id, downlink.down_id)
222
+ manager.remove_link(uplink.up_id, uplink.down_id)
223
+ manager.remove_link(downlink.up_id, downlink.down_id)
227
224
  output new_link.to_unix
228
225
  end
229
226
  end
230
- service.delete_shard shard_id
227
+ manager.delete_shard shard_id
231
228
  end
232
229
  end
233
230
  end
234
231
 
235
- class CreateCommand < ShardCommand
232
+ class CreateCommand < Command
236
233
  def run
237
234
  help! if argv.length < 2
238
235
  class_name, *shard_ids = argv
@@ -241,26 +238,26 @@ module Gizzard
241
238
  destination_type = command_options.destination_type || ""
242
239
  shard_ids.each do |id|
243
240
  shard_id = ShardId.parse(id)
244
- service.create_shard(ShardInfo.new(shard_id, class_name, source_type, destination_type, busy))
245
- service.get_shard(shard_id)
241
+ manager.create_shard(ShardInfo.new(shard_id, class_name, source_type, destination_type, busy))
242
+ manager.get_shard(shard_id)
246
243
  output shard_id.to_unix
247
244
  end
248
245
  end
249
246
  end
250
247
 
251
- class LinksCommand < ShardCommand
248
+ class LinksCommand < Command
252
249
  def run
253
250
  shard_ids = @argv
254
251
  shard_ids.each do |shard_id_text|
255
252
  shard_id = ShardId.parse(shard_id_text)
256
253
  next if !shard_id
257
254
  unless command_options.down
258
- service.list_upward_links(shard_id).each do |link_info|
255
+ manager.list_upward_links(shard_id).each do |link_info|
259
256
  output command_options.ids ? link_info.up_id.to_unix : link_info.to_unix
260
257
  end
261
258
  end
262
259
  unless command_options.up
263
- service.list_downward_links(shard_id).each do |link_info|
260
+ manager.list_downward_links(shard_id).each do |link_info|
264
261
  output command_options.ids ? link_info.down_id.to_unix : link_info.to_unix
265
262
  end
266
263
  end
@@ -268,41 +265,41 @@ module Gizzard
268
265
  end
269
266
  end
270
267
 
271
- class InfoCommand < ShardCommand
268
+ class InfoCommand < Command
272
269
  def run
273
270
  shard_ids = @argv
274
271
  shard_ids.each do |shard_id|
275
- shard_info = service.get_shard(ShardId.parse(shard_id))
272
+ shard_info = manager.get_shard(ShardId.parse(shard_id))
276
273
  output shard_info.to_unix
277
274
  end
278
275
  end
279
276
  end
280
277
 
281
- class MarkbusyCommand < ShardCommand
278
+ class MarkbusyCommand < Command
282
279
  def run
283
280
  shard_ids = @argv
284
281
  shard_ids.each do |shard_id|
285
282
  id = ShardId.parse(shard_id)
286
- service.mark_shard_busy(id, 1)
287
- shard_info = service.get_shard(id)
283
+ manager.mark_shard_busy(id, 1)
284
+ shard_info = manager.get_shard(id)
288
285
  output shard_info.to_unix
289
286
  end
290
287
  end
291
288
  end
292
289
 
293
- class MarkunbusyCommand < ShardCommand
290
+ class MarkunbusyCommand < Command
294
291
  def run
295
292
  shard_ids = @argv
296
293
  shard_ids.each do |shard_id|
297
294
  id = ShardId.parse(shard_id)
298
- service.mark_shard_busy(id, 0)
299
- shard_info = service.get_shard(id)
295
+ manager.mark_shard_busy(id, 0)
296
+ shard_info = manager.get_shard(id)
300
297
  output shard_info.to_unix
301
298
  end
302
299
  end
303
300
  end
304
301
 
305
- class RepairCommand < ShardCommand
302
+ class RepairCommand < Command
306
303
  def run
307
304
  args = @argv.dup.map{|a| a.split(/\s+/)}.flatten
308
305
  pairs = []
@@ -314,8 +311,8 @@ module Gizzard
314
311
  end
315
312
  pairs.each do |master, slave|
316
313
  puts "#{master} #{slave}"
317
- mprefixes = service.shards_for_hostname(master).map{|s| s.id.table_prefix}
318
- sprefixes = service.shards_for_hostname(slave).map{|s| s.id.table_prefix}
314
+ mprefixes = manager.shards_for_hostname(master).map{|s| s.id.table_prefix}
315
+ sprefixes = manager.shards_for_hostname(slave).map{|s| s.id.table_prefix}
319
316
  delta = mprefixes - sprefixes
320
317
  delta.each do |prefix|
321
318
  puts "gizzmo copy #{master}/#{prefix} #{slave}/#{prefix}"
@@ -324,7 +321,7 @@ module Gizzard
324
321
  end
325
322
  end
326
323
 
327
- class WrapCommand < ShardCommand
324
+ class WrapCommand < Command
328
325
  def self.derive_wrapper_shard_id(shard_info, wrapping_class_name)
329
326
  suffix = "_" + wrapping_class_name.split(".").last.downcase.gsub("shard", "")
330
327
  ShardId.new("localhost", shard_info.id.table_prefix + suffix)
@@ -335,15 +332,15 @@ module Gizzard
335
332
  help! "No shards specified" if shard_ids.empty?
336
333
  shard_ids.each do |shard_id_string|
337
334
  shard_id = ShardId.parse(shard_id_string)
338
- shard_info = service.get_shard(shard_id)
339
- service.create_shard(ShardInfo.new(wrapper_id = self.class.derive_wrapper_shard_id(shard_info, class_name), class_name, "", "", 0))
335
+ shard_info = manager.get_shard(shard_id)
336
+ manager.create_shard(ShardInfo.new(wrapper_id = self.class.derive_wrapper_shard_id(shard_info, class_name), class_name, "", "", 0))
340
337
 
341
- existing_links = service.list_upward_links(shard_id)
338
+ existing_links = manager.list_upward_links(shard_id)
342
339
  unless existing_links.include?(LinkInfo.new(wrapper_id, shard_id, 1))
343
- service.add_link(wrapper_id, shard_id, 1)
340
+ manager.add_link(wrapper_id, shard_id, 1)
344
341
  existing_links.each do |link_info|
345
- service.add_link(link_info.up_id, wrapper_id, link_info.weight)
346
- service.remove_link(link_info.up_id, link_info.down_id)
342
+ manager.add_link(link_info.up_id, wrapper_id, link_info.weight)
343
+ manager.remove_link(link_info.up_id, link_info.down_id)
347
344
  end
348
345
  end
349
346
  output wrapper_id.to_unix
@@ -351,7 +348,7 @@ module Gizzard
351
348
  end
352
349
  end
353
350
 
354
- class RebalanceCommand < ShardCommand
351
+ class RebalanceCommand < Command
355
352
 
356
353
  class NamedArray < Array
357
354
  attr_reader :name
@@ -413,7 +410,7 @@ module Gizzard
413
410
  host = set.name
414
411
  set.each do |id|
415
412
  if id.hostname != host
416
- shard_info ||= service.get_shard(id)
413
+ shard_info ||= manager.get_shard(id)
417
414
  old = id.to_unix
418
415
  id.hostname = host
419
416
  shards << [old, id.to_unix]
@@ -429,11 +426,11 @@ module Gizzard
429
426
  end
430
427
  end
431
428
 
432
- class PairCommand < ShardCommand
429
+ class PairCommand < Command
433
430
  def run
434
431
  ids = []
435
432
  @argv.map do |host|
436
- service.shards_for_hostname(host).each do |shard|
433
+ manager.shards_for_hostname(host).each do |shard|
437
434
  ids << shard.id
438
435
  end
439
436
  end
@@ -460,11 +457,11 @@ module Gizzard
460
457
  displayed = {}
461
458
  overlaps.sort_by { |hosts, count| count }.reverse.each do |(host_a, host_b), count|
462
459
  next if !host_a || !host_b || displayed[host_a] || displayed[host_b]
463
- id_a = ids_by_host[host_a].find { |id| service.list_upward_links(id).size > 0 }
464
- id_b = ids_by_host[host_b].find { |id| service.list_upward_links(id).size > 0 }
460
+ id_a = ids_by_host[host_a].find {|id| manager.list_upward_links(id).size > 0 }
461
+ id_b = ids_by_host[host_b].find {|id| manager.list_upward_links(id).size > 0 }
465
462
  next unless id_a && id_b
466
- weight_a = service.list_upward_links(id_a).first.weight
467
- weight_b = service.list_upward_links(id_b).first.weight
463
+ weight_a = manager.list_upward_links(id_a).first.weight
464
+ weight_b = manager.list_upward_links(id_b).first.weight
468
465
  if weight_a > weight_b
469
466
  puts "#{host_a}\t#{host_b}"
470
467
  else
@@ -483,7 +480,7 @@ module Gizzard
483
480
  end
484
481
  end
485
482
 
486
- class ReportCommand < ShardCommand
483
+ class ReportCommand < Command
487
484
  def run
488
485
  things = @argv.map do |shard|
489
486
  parse(down(ShardId.parse(shard))).join("\n")
@@ -532,7 +529,7 @@ module Gizzard
532
529
  end
533
530
 
534
531
  def down(id)
535
- vals = service.list_downward_links(id).map do |link|
532
+ vals = manager.list_downward_links(id).map do |link|
536
533
  down(link.down_id)
537
534
  end
538
535
  { id.to_unix => vals }
@@ -550,11 +547,11 @@ module Gizzard
550
547
  end
551
548
  end
552
549
 
553
- class FindCommand < ShardCommand
550
+ class FindCommand < Command
554
551
  def run
555
552
  hosts = @argv << command_options.shard_host
556
553
  hosts.compact.each do |host|
557
- service.shards_for_hostname(host).each do |shard|
554
+ manager.shards_for_hostname(host).each do |shard|
558
555
  next if command_options.shard_type && shard.class_name !~ Regexp.new(command_options.shard_type)
559
556
  output shard.id.to_unix
560
557
  end
@@ -562,7 +559,7 @@ module Gizzard
562
559
  end
563
560
  end
564
561
 
565
- class LookupCommand < ShardCommand
562
+ class LookupCommand < Command
566
563
  def run
567
564
  table_id, source = @argv
568
565
  help!("Requires table id and source") unless table_id && source
@@ -572,51 +569,51 @@ module Gizzard
572
569
  else
573
570
  source_id = source.to_i
574
571
  end
575
- shard = service.find_current_forwarding(table_id.to_i, source_id)
572
+ shard = manager.find_current_forwarding(table_id.to_i, source_id)
576
573
  output shard.id.to_unix
577
574
  end
578
575
  end
579
576
 
580
- class CopyCommand < ShardCommand
577
+ class CopyCommand < Command
581
578
  def run
582
579
  from_shard_id_string, to_shard_id_string = @argv
583
580
  help!("Requires source & destination shard id") unless from_shard_id_string && to_shard_id_string
584
581
  from_shard_id = ShardId.parse(from_shard_id_string)
585
582
  to_shard_id = ShardId.parse(to_shard_id_string)
586
- service.copy_shard(from_shard_id, to_shard_id)
583
+ manager.copy_shard(from_shard_id, to_shard_id)
587
584
  end
588
585
  end
589
586
 
590
- class BusyCommand < ShardCommand
587
+ class BusyCommand < Command
591
588
  def run
592
- service.get_busy_shards().each { |shard_info| output shard_info.to_unix }
589
+ manager.get_busy_shards().each { |shard_info| output shard_info.to_unix }
593
590
  end
594
591
  end
595
592
 
596
- class SetupReplicaCommand < ShardCommand
593
+ class SetupReplicaCommand < Command
597
594
  def run
598
595
  from_shard_id_string, to_shard_id_string = @argv
599
596
  help!("Requires source & destination shard id") unless from_shard_id_string && to_shard_id_string
600
597
  from_shard_id = ShardId.parse(from_shard_id_string)
601
598
  to_shard_id = ShardId.parse(to_shard_id_string)
602
599
 
603
- if service.list_upward_links(to_shard_id).size > 0
600
+ if manager.list_upward_links(to_shard_id).size > 0
604
601
  STDERR.puts "Destination shard #{to_shard_id} has links to it."
605
602
  exit 1
606
603
  end
607
604
 
608
- link = service.list_upward_links(from_shard_id)[0]
605
+ link = manager.list_upward_links(from_shard_id)[0]
609
606
  replica_shard_id = link.up_id
610
607
  weight = link.weight
611
608
  write_only_shard_id = ShardId.new("localhost", "#{to_shard_id.table_prefix}_copy_write_only")
612
- service.create_shard(ShardInfo.new(write_only_shard_id, "WriteOnlyShard", "", "", 0))
613
- service.add_link(replica_shard_id, write_only_shard_id, weight)
614
- service.add_link(write_only_shard_id, to_shard_id, 1)
609
+ manager.create_shard(ShardInfo.new(write_only_shard_id, "WriteOnlyShard", "", "", 0))
610
+ manager.add_link(replica_shard_id, write_only_shard_id, weight)
611
+ manager.add_link(write_only_shard_id, to_shard_id, 1)
615
612
  output to_shard_id.to_unix
616
613
  end
617
614
  end
618
615
 
619
- class FinishReplicaCommand < ShardCommand
616
+ class FinishReplicaCommand < Command
620
617
  def run
621
618
  from_shard_id_string, to_shard_id_string = @argv
622
619
  help!("Requires source & destination shard id") unless from_shard_id_string && to_shard_id_string
@@ -624,58 +621,58 @@ module Gizzard
624
621
  to_shard_id = ShardId.parse(to_shard_id_string)
625
622
 
626
623
  write_only_shard_id = ShardId.new("localhost", "#{to_shard_id.table_prefix}_copy_write_only")
627
- link = service.list_upward_links(write_only_shard_id)[0]
624
+ link = manager.list_upward_links(write_only_shard_id)[0]
628
625
  replica_shard_id = link.up_id
629
626
  weight = link.weight
630
627
 
631
628
  # careful. need to validate some basic assumptions.
632
629
  unless global_options.force
633
- if service.list_upward_links(from_shard_id).map { |link| link.up_id }.to_a != [ replica_shard_id ]
630
+ if manager.list_upward_links(from_shard_id).map { |link| link.up_id }.to_a != [ replica_shard_id ]
634
631
  STDERR.puts "Uplink from #{from_shard_id} is not a migration replica."
635
632
  exit 1
636
633
  end
637
- if service.list_upward_links(to_shard_id).map { |link| link.up_id }.to_a != [ write_only_shard_id ]
634
+ if manager.list_upward_links(to_shard_id).map { |link| link.up_id }.to_a != [ write_only_shard_id ]
638
635
  STDERR.puts "Uplink from #{to_shard_id} is not a write-only barrier."
639
636
  exit 1
640
637
  end
641
638
  end
642
639
 
643
- service.remove_link(write_only_shard_id, to_shard_id)
644
- service.remove_link(replica_shard_id, write_only_shard_id)
645
- service.add_link(replica_shard_id, to_shard_id, weight)
646
- service.delete_shard(write_only_shard_id)
640
+ manager.remove_link(write_only_shard_id, to_shard_id)
641
+ manager.remove_link(replica_shard_id, write_only_shard_id)
642
+ manager.add_link(replica_shard_id, to_shard_id, weight)
643
+ manager.delete_shard(write_only_shard_id)
647
644
  end
648
645
  end
649
646
 
650
- class SetupMigrateCommand < ShardCommand
647
+ class SetupMigrateCommand < Command
651
648
  def run
652
649
  from_shard_id_string, to_shard_id_string = @argv
653
650
  help!("Requires source & destination shard id") unless from_shard_id_string && to_shard_id_string
654
651
  from_shard_id = ShardId.parse(from_shard_id_string)
655
652
  to_shard_id = ShardId.parse(to_shard_id_string)
656
653
 
657
- if service.list_upward_links(to_shard_id).size > 0
654
+ if manager.list_upward_links(to_shard_id).size > 0
658
655
  STDERR.puts "Destination shard #{to_shard_id} has links to it."
659
656
  exit 1
660
657
  end
661
658
 
662
659
  write_only_shard_id = ShardId.new("localhost", "#{to_shard_id.table_prefix}_migrate_write_only")
663
660
  replica_shard_id = ShardId.new("localhost", "#{to_shard_id.table_prefix}_migrate_replica")
664
- service.create_shard(ShardInfo.new(write_only_shard_id, "com.twitter.gizzard.shards.WriteOnlyShard", "", "", 0))
665
- service.create_shard(ShardInfo.new(replica_shard_id, "com.twitter.gizzard.shards.ReplicatingShard", "", "", 0))
666
- service.add_link(write_only_shard_id, to_shard_id, 1)
667
- service.list_upward_links(from_shard_id).each do |link|
668
- service.remove_link(link.up_id, link.down_id)
669
- service.add_link(link.up_id, replica_shard_id, link.weight)
670
- end
671
- service.add_link(replica_shard_id, from_shard_id, 1)
672
- service.add_link(replica_shard_id, write_only_shard_id, 0)
673
- service.replace_forwarding(from_shard_id, replica_shard_id)
661
+ manager.create_shard(ShardInfo.new(write_only_shard_id, "com.twitter.gizzard.shards.WriteOnlyShard", "", "", 0))
662
+ manager.create_shard(ShardInfo.new(replica_shard_id, "com.twitter.gizzard.shards.ReplicatingShard", "", "", 0))
663
+ manager.add_link(write_only_shard_id, to_shard_id, 1)
664
+ manager.list_upward_links(from_shard_id).each do |link|
665
+ manager.remove_link(link.up_id, link.down_id)
666
+ manager.add_link(link.up_id, replica_shard_id, link.weight)
667
+ end
668
+ manager.add_link(replica_shard_id, from_shard_id, 1)
669
+ manager.add_link(replica_shard_id, write_only_shard_id, 0)
670
+ manager.replace_forwarding(from_shard_id, replica_shard_id)
674
671
  output replica_shard_id.to_unix
675
672
  end
676
673
  end
677
674
 
678
- class FinishMigrateCommand < ShardCommand
675
+ class FinishMigrateCommand < Command
679
676
  def run
680
677
  from_shard_id_string, to_shard_id_string = @argv
681
678
  help!("Requires source & destination shard id") unless from_shard_id_string && to_shard_id_string
@@ -687,57 +684,194 @@ module Gizzard
687
684
 
688
685
  # careful. need to validate some basic assumptions.
689
686
  unless global_options.force
690
- if service.list_upward_links(from_shard_id).map { |link| link.up_id }.to_a != [ replica_shard_id ]
687
+ if manager.list_upward_links(from_shard_id).map { |link| link.up_id }.to_a != [ replica_shard_id ]
691
688
  STDERR.puts "Uplink from #{from_shard_id} is not a migration replica."
692
689
  exit 1
693
690
  end
694
- if service.list_upward_links(to_shard_id).map { |link| link.up_id }.to_a != [ write_only_shard_id ]
691
+ if manager.list_upward_links(to_shard_id).map { |link| link.up_id }.to_a != [ write_only_shard_id ]
695
692
  STDERR.puts "Uplink from #{to_shard_id} is not a write-only barrier."
696
693
  exit 1
697
694
  end
698
- if service.list_upward_links(write_only_shard_id).map { |link| link.up_id }.to_a != [ replica_shard_id ]
695
+ if manager.list_upward_links(write_only_shard_id).map { |link| link.up_id }.to_a != [ replica_shard_id ]
699
696
  STDERR.puts "Uplink from write-only barrier is not a migration replica."
700
697
  exit 1
701
698
  end
702
699
  end
703
700
 
704
- service.remove_link(write_only_shard_id, to_shard_id)
705
- service.list_upward_links(replica_shard_id).each do |link|
706
- service.remove_link(link.up_id, link.down_id)
707
- service.add_link(link.up_id, to_shard_id, link.weight)
701
+ manager.remove_link(write_only_shard_id, to_shard_id)
702
+ manager.list_upward_links(replica_shard_id).each do |link|
703
+ manager.remove_link(link.up_id, link.down_id)
704
+ manager.add_link(link.up_id, to_shard_id, link.weight)
708
705
  end
709
- service.replace_forwarding(replica_shard_id, to_shard_id)
710
- service.delete_shard(replica_shard_id)
711
- service.delete_shard(write_only_shard_id)
706
+ manager.replace_forwarding(replica_shard_id, to_shard_id)
707
+ manager.delete_shard(replica_shard_id)
708
+ manager.delete_shard(write_only_shard_id)
712
709
  end
713
710
  end
714
711
 
715
- class InjectCommand < JobCommand
712
+ class InjectCommand < Command
716
713
  def run
714
+ count = 0
715
+ page_size = 20
717
716
  priority, *jobs = @argv
718
717
  help!("Requires priority") unless priority and jobs.size > 0
719
- count = 0
720
- jobs.each do |job|
721
- service.inject_job(priority.to_i, job)
718
+
719
+ jobs.each_slice(page_size) do |js|
720
+ job_injector.inject_jobs(js.map {|j| Job.new(priority.to_i, j) })
721
+
722
722
  count += 1
723
723
  # FIXME add -q --quiet option
724
724
  STDERR.print "."
725
- STDERR.print "#{count}" if count % 100 == 0
725
+ STDERR.print "#{count * page_size}" if count % 10 == 0
726
726
  STDERR.flush
727
727
  end
728
728
  STDERR.print "\n"
729
729
  end
730
730
  end
731
731
 
732
- class FlushCommand < JobCommand
732
+ class FlushCommand < Command
733
733
  def run
734
734
  args = @argv[0]
735
735
  help!("Requires --all, or a job priority id.") unless args || command_options.flush_all
736
736
  if command_options.flush_all
737
- service.retry_errors()
737
+ manager.retry_errors()
738
738
  else
739
- service.retry_errors_for(args.to_i)
739
+ manager.retry_errors_for(args.to_i)
740
+ end
741
+ end
742
+ end
743
+
744
+
745
+ class AddHostCommand < Command
746
+ def run
747
+ hosts = @argv.map do |arg|
748
+ cluster, hostname, port = *arg.split(":")
749
+ help!("malformed host argument") unless [cluster, hostname, port].compact.length == 3
750
+
751
+ Host.new(hostname, port.to_i, cluster, HostStatus::Normal)
740
752
  end
753
+
754
+ hosts.each {|h| manager.add_remote_host(h) }
755
+ end
756
+ end
757
+
758
+ class RemoveHostCommand < Command
759
+ def run
760
+ host = @argv[0].split(":")
761
+ host.unshift nil if host.length == 2
762
+ cluster, hostname, port = *host
763
+
764
+ manager.remove_remote_host(hostname, port.to_i)
765
+ end
766
+ end
767
+
768
+ class ListHostsCommand < Command
769
+ def run
770
+ manager.list_remote_hosts.each do |host|
771
+ puts "#{[host.cluster, host.hostname, host.port].join(":")} #{host.status}"
772
+ end
773
+ end
774
+ end
775
+
776
+ class TopologyCommand < Command
777
+ def run
778
+ templates = manager.manifest(*global_options.tables).templates.inject({}) do |h, (t, fs)|
779
+ h.update t.to_config => fs
780
+ end
781
+
782
+ if command_options.forwardings
783
+ templates.
784
+ inject([]) { |h, (t, fs)| fs.each { |f| h << [f.base_id, t] }; h }.
785
+ sort.
786
+ each { |a| puts "%25d\t%s" % a }
787
+ else
788
+ templates.
789
+ map { |(t, fs)| [fs.length, t] }.
790
+ sort.reverse.
791
+ each { |a| puts "%4d %s" % a }
792
+ end
793
+ end
794
+ end
795
+
796
+ class TransformTreeCommand < Command
797
+ def run
798
+ help!("wrong number of arguments") unless @argv.length == 2
799
+
800
+ scheduler_options = command_options.scheduler_options || {}
801
+ template_s, shard_id_s = @argv
802
+
803
+ to_template = ShardTemplate.parse(template_s)
804
+ shard_id = ShardId.parse(shard_id_s)
805
+ base_name = shard_id.table_prefix.split('_').first
806
+ forwarding = manager.get_forwarding_for_shard(shard_id)
807
+ manifest = manager.manifest(forwarding.table_id)
808
+ shard = manifest.trees[forwarding]
809
+ copy_wrapper = scheduler_options[:copy_wrapper]
810
+ be_quiet = global_options.force && command_options.quiet
811
+ transformation = Transformation.new(shard.template, to_template, copy_wrapper)
812
+
813
+ scheduler_options[:quiet] = be_quiet
814
+
815
+ unless be_quiet
816
+ puts transformation.inspect
817
+ puts ""
818
+ end
819
+
820
+ unless global_options.force
821
+ print "Continue? (y/n) "; $stdout.flush
822
+ exit unless $stdin.gets.chomp == "y"
823
+ puts ""
824
+ end
825
+
826
+ Gizzard.schedule! manager,
827
+ base_name,
828
+ { transformation => { forwarding => shard } },
829
+ scheduler_options
830
+ end
831
+ end
832
+
833
+ class TransformCommand < Command
834
+ def run
835
+ help!("must have an even number of arguments") unless @argv.length % 2 == 0
836
+
837
+ scheduler_options = command_options.scheduler_options || {}
838
+ manifest = manager.manifest(*global_options.tables)
839
+ copy_wrapper = scheduler_options[:copy_wrapper]
840
+ be_quiet = global_options.force && command_options.quiet
841
+ transformations = {}
842
+
843
+ scheduler_options[:quiet] = be_quiet
844
+
845
+ @argv.each_slice(2) do |(from_template_s, to_template_s)|
846
+ from, to = [from_template_s, to_template_s].map {|s| ShardTemplate.parse(s) }
847
+ transformation = Transformation.new(from, to, copy_wrapper)
848
+ forwardings = Set.new(manifest.templates[from] || [])
849
+ trees = manifest.trees.reject {|(f, s)| !forwardings.include?(f) }
850
+
851
+ transformations[transformation] = trees
852
+ end
853
+
854
+ base_name = transformations.values.first.values.first.id.table_prefix.split('_').first
855
+
856
+ unless be_quiet
857
+ transformations.each do |transformation, trees|
858
+ puts transformation.inspect
859
+ puts "Applied to #{trees.length} shards:"
860
+ trees.keys.sort.each {|f| puts " #{f.inspect}" }
861
+ end
862
+ puts ""
863
+ end
864
+
865
+ unless global_options.force
866
+ print "Continue? (y/n) "; $stdout.flush
867
+ exit unless $stdin.gets.chomp == "y"
868
+ puts ""
869
+ end
870
+
871
+ Gizzard.schedule! manager,
872
+ base_name,
873
+ transformations,
874
+ scheduler_options
741
875
  end
742
876
  end
743
877
  end