gizzmo 0.11.0 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Rakefile +11 -16
- data/VERSION +1 -1
- data/bin/setup_shards +173 -0
- data/lib/gizzard.rb +4 -0
- data/lib/gizzard/commands.rb +286 -152
- data/lib/gizzard/migrator.rb +192 -0
- data/lib/gizzard/nameserver.rb +206 -0
- data/lib/gizzard/shard_template.rb +252 -0
- data/lib/gizzard/thrift.rb +187 -135
- data/lib/gizzard/transformation.rb +230 -0
- data/lib/gizzard/transformation_op.rb +181 -0
- data/lib/gizzard/transformation_scheduler.rb +220 -0
- data/lib/gizzmo.rb +87 -20
- data/test/gizzmo_spec.rb +499 -0
- data/test/nameserver_spec.rb +139 -0
- data/test/scheduler_spec.rb +59 -0
- data/test/shard_template_spec.rb +103 -0
- data/test/spec.opts +7 -0
- data/test/spec_helper.rb +139 -0
- data/test/test_server/.gitignore +13 -0
- data/test/test_server/project/build.properties +8 -0
- data/test/test_server/project/build/Project.scala +13 -0
- data/test/test_server/project/plugins/Plugins.scala +6 -0
- data/test/test_server/src/main/scala/Main.scala +18 -0
- data/test/test_server/src/main/scala/TestServer.scala +247 -0
- data/test/test_server/src/main/thrift/TestServer.thrift +12 -0
- data/test/transformation_spec.rb +181 -0
- metadata +32 -5
- data/gizzmo.gemspec +0 -75
data/Rakefile
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
ROOT_DIR = File.expand_path(File.dirname(__FILE__))
|
1
2
|
require 'rubygems'
|
2
3
|
require 'rake'
|
3
4
|
|
@@ -16,28 +17,22 @@ rescue LoadError
|
|
16
17
|
puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
|
17
18
|
end
|
18
19
|
|
19
|
-
require 'rake/testtask'
|
20
|
-
Rake::TestTask.new(:test) do |test|
|
21
|
-
test.libs << 'lib' << 'test'
|
22
|
-
test.pattern = 'test/**/test_*.rb'
|
23
|
-
test.verbose = true
|
24
|
-
end
|
25
20
|
|
26
21
|
begin
|
27
|
-
require '
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
22
|
+
require 'spec/rake/spectask'
|
23
|
+
Spec::Rake::SpecTask.new(:spec) do |t|
|
24
|
+
spec_opts = File.expand_path('test/spec.opts', ROOT_DIR)
|
25
|
+
if File.exist? spec_opts
|
26
|
+
t.spec_opts = ['--options', "\"#{spec_opts}\""]
|
27
|
+
end
|
28
|
+
t.spec_files = FileList['test/**/*_spec.rb']
|
32
29
|
end
|
33
30
|
rescue LoadError
|
34
|
-
|
35
|
-
abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
|
36
|
-
end
|
31
|
+
$stderr.puts "RSpec required to run tests."
|
37
32
|
end
|
38
33
|
|
39
34
|
task :test do
|
40
|
-
puts
|
35
|
+
puts
|
41
36
|
puts "=" * 79
|
42
37
|
puts "You might want to read the README before running tests."
|
43
38
|
puts "=" * 79
|
@@ -45,7 +40,7 @@ task :test do
|
|
45
40
|
exec File.join(File.dirname(__FILE__), "test", "test.sh")
|
46
41
|
end
|
47
42
|
|
48
|
-
task :default => :
|
43
|
+
task :default => :spec
|
49
44
|
|
50
45
|
require 'rake/rdoctask'
|
51
46
|
Rake::RDocTask.new do |rdoc|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.11.
|
1
|
+
0.11.1
|
data/bin/setup_shards
ADDED
@@ -0,0 +1,173 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'yaml'
|
4
|
+
require 'ostruct'
|
5
|
+
require 'optparse'
|
6
|
+
require 'gizzard'
|
7
|
+
|
8
|
+
require 'gizzard/nameserver'
|
9
|
+
require 'gizzard/migrator'
|
10
|
+
require 'gizzard/transformation'
|
11
|
+
require 'gizzard/shard_template'
|
12
|
+
|
13
|
+
def usage(parser, reason)
|
14
|
+
puts
|
15
|
+
puts "ERROR: #{reason}"
|
16
|
+
puts
|
17
|
+
puts parser
|
18
|
+
puts
|
19
|
+
exit 1
|
20
|
+
end
|
21
|
+
|
22
|
+
options = {
|
23
|
+
:config_filename => ENV['HOME'] + "/.topology.yml",
|
24
|
+
:dry_run => false,
|
25
|
+
:max_copies => 20,
|
26
|
+
:table_id => nil
|
27
|
+
}
|
28
|
+
|
29
|
+
parser = OptionParser.new do |opts|
|
30
|
+
opts.banner = "Usage: #{$0} [options]"
|
31
|
+
opts.separator "Example: #{$0} -f topology.yml"
|
32
|
+
|
33
|
+
opts.on("-f", "--config=FILENAME", "load database topology config (default: #{options[:config_filename]})") do |filename|
|
34
|
+
options[:config_filename] = filename
|
35
|
+
end
|
36
|
+
opts.on("-i", "--id=TABLE_ID", "table id to use (default: #{options[:table_id]})") do |table_id|
|
37
|
+
options[:table_id] = table_id.to_i
|
38
|
+
end
|
39
|
+
opts.on("-s", "--shards=N", "create N shards") do |n|
|
40
|
+
options[:total_shards] = n.to_i
|
41
|
+
end
|
42
|
+
opts.on("-t", "--table=TABLE_NAME", "table name") do |table_name|
|
43
|
+
options[:table_name] = table_name
|
44
|
+
end
|
45
|
+
opts.on("-c", "--class=CLASS_NAME", "shard class to create") do |class_name|
|
46
|
+
options[:shard_class] = class_name
|
47
|
+
end
|
48
|
+
opts.on("-n", "--dry-run", "don't actually send gizzard commands") do
|
49
|
+
options[:dry_run] = true
|
50
|
+
end
|
51
|
+
opts.on("-x", "--max=COPIES", "max concurrent copies (default: #{options[:max_copies]})") do |n|
|
52
|
+
options[:max_copies] = n.to_i
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
parser.parse!(ARGV)
|
57
|
+
|
58
|
+
config = begin
|
59
|
+
YAML.load_file(options[:config_filename])
|
60
|
+
rescue => e
|
61
|
+
puts "Exception while reading config file: #{e}"
|
62
|
+
{}
|
63
|
+
end
|
64
|
+
|
65
|
+
total_shards = options[:total_shards] || config['total_shards']
|
66
|
+
usage(parser, "Must define total_shards or -s") unless total_shards
|
67
|
+
|
68
|
+
table_name = options[:table_name] || config['table_name']
|
69
|
+
usage(parser, "Must define table_name or -t") unless table_name
|
70
|
+
|
71
|
+
migrator_config = Gizzard::MigratorConfig.new
|
72
|
+
migrator_config.prefix = table_name
|
73
|
+
migrator_config.table_id = options[:table_id]
|
74
|
+
migrator_config.source_type = config['source_type'] || ''
|
75
|
+
migrator_config.destination_type = config['destination_type'] || ''
|
76
|
+
migrator_config.forwarding_space = config['forwarding_space'] || 2 ** 64
|
77
|
+
migrator_config.forwarding_space_min = config['forwarding_space_min'] || 2 ** 63 * -1
|
78
|
+
|
79
|
+
querying_nameserver = Gizzard::Nameserver.new(config['app_hosts'] || 'localhost', :dry_run => false)
|
80
|
+
mutating_nameserver = Gizzard::Nameserver.new(config['app_hosts'] || 'localhost', :dry_run => options[:dry_run])
|
81
|
+
|
82
|
+
usage(parser, "Config file must contain a 'partitions' definition") unless config['partitions']
|
83
|
+
|
84
|
+
# Where the magic happens:
|
85
|
+
|
86
|
+
new_templates = config['partitions'].map { |p| Gizzard::ShardTemplate.from_config(migrator_config, p) }
|
87
|
+
|
88
|
+
EXISTING_CONFIG_CACHE = File.expand_path(".existing_shards.marshal")
|
89
|
+
|
90
|
+
if File.exist? EXISTING_CONFIG_CACHE
|
91
|
+
puts "Loading cache from previous operation. If this is not desired, delete the cache file at #{EXISTING_CONFIG_CACHE}"
|
92
|
+
manifest = Marshal.load(File.read(EXISTING_CONFIG_CACHE))
|
93
|
+
raise "error loading cache!" unless manifest.is_a? Gizzard::Manifest
|
94
|
+
else
|
95
|
+
puts "Querying nameserver..."
|
96
|
+
manifest = Gizzard::Manifest.new(querying_nameserver, migrator_config)
|
97
|
+
File.open(EXISTING_CONFIG_CACHE, 'w') { |f| f.write Marshal.dump(manifest) }
|
98
|
+
end
|
99
|
+
|
100
|
+
migrator_config.manifest = manifest
|
101
|
+
existing_map = manifest.template_map
|
102
|
+
|
103
|
+
|
104
|
+
puts "\nCalculating changes...\n"
|
105
|
+
|
106
|
+
|
107
|
+
migrator = Gizzard::Migrator.new(existing_map, new_templates, total_shards, migrator_config)
|
108
|
+
|
109
|
+
puts "\nUNCHANGED:"
|
110
|
+
pp migrator.unchanged_templates
|
111
|
+
|
112
|
+
puts "\nSIMILAR:"
|
113
|
+
pp migrator.similar_templates
|
114
|
+
|
115
|
+
puts "\nNEW:"
|
116
|
+
pp migrator.new_templates
|
117
|
+
|
118
|
+
puts "\nUNRECOGNIZED:"
|
119
|
+
pp migrator.unrecognized_templates
|
120
|
+
|
121
|
+
unless (transformations = migrator.transformations).empty?
|
122
|
+
puts "\nTRANSFORMATIONS:"
|
123
|
+
transformations.each { |t| puts ""; p t }
|
124
|
+
|
125
|
+
printf "Apply migration? [yN] "; $stdout.flush
|
126
|
+
|
127
|
+
if $stdin.gets.chomp == 'y'
|
128
|
+
pages = transformations.inject([]) { |pages, t| pages.concat t.paginate(options[:max_copies]) }
|
129
|
+
|
130
|
+
pages.each_with_index do |page, page_idx|
|
131
|
+
puts "\nTRANSFORMATION #{page_idx + 1} / #{pages.length}:"
|
132
|
+
|
133
|
+
puts page.inspect(true)
|
134
|
+
|
135
|
+
t_start = Time.now
|
136
|
+
|
137
|
+
if page.must_copy?
|
138
|
+
puts " Preparing nameserver for copies."
|
139
|
+
page.prepare! mutating_nameserver, migrator_config
|
140
|
+
mutating_nameserver.reload_forwardings
|
141
|
+
|
142
|
+
puts " Scheduling copies."
|
143
|
+
page.copy! mutating_nameserver, migrator_config
|
144
|
+
|
145
|
+
puts " Waiting for copies to finish."
|
146
|
+
page.wait_for_copies mutating_nameserver, migrator_config
|
147
|
+
|
148
|
+
puts " Finalizing nameserver changes."
|
149
|
+
page.cleanup! mutating_nameserver, migrator_config
|
150
|
+
mutating_nameserver.reload_forwardings
|
151
|
+
else
|
152
|
+
puts " Applying nameserver changes."
|
153
|
+
page.apply! mutating_nameserver, migrator_config
|
154
|
+
end
|
155
|
+
|
156
|
+
puts "--- Time Elapsed: %0.3f ---" % (Time.now - t_start).to_f
|
157
|
+
end
|
158
|
+
|
159
|
+
mutating_nameserver.reload_forwardings
|
160
|
+
|
161
|
+
puts "Done!"
|
162
|
+
|
163
|
+
else
|
164
|
+
puts "Aborting migration."
|
165
|
+
end
|
166
|
+
|
167
|
+
else
|
168
|
+
puts "No migration needed."
|
169
|
+
end
|
170
|
+
|
171
|
+
# remove the existing config state since we're done, and the system is
|
172
|
+
# in a known good state.
|
173
|
+
File.unlink EXISTING_CONFIG_CACHE
|
data/lib/gizzard.rb
CHANGED
data/lib/gizzard/commands.rb
CHANGED
@@ -1,29 +1,51 @@
|
|
1
1
|
require "pp"
|
2
|
+
require "set"
|
2
3
|
require "digest/md5"
|
3
4
|
|
4
5
|
module Gizzard
|
5
6
|
class Command
|
6
|
-
include Thrift
|
7
7
|
|
8
8
|
attr_reader :buffer
|
9
9
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
10
|
+
class << self
|
11
|
+
def run(command_name, global_options, argv, subcommand_options, log)
|
12
|
+
command_class = Gizzard.const_get("#{classify(command_name)}Command")
|
13
|
+
|
14
|
+
@manager ||= make_manager(global_options, log)
|
15
|
+
@job_injector ||= make_job_injector(global_options, log)
|
16
|
+
|
17
|
+
command = command_class.new(@manager, @job_injector, global_options, argv, subcommand_options)
|
18
|
+
command.run
|
19
|
+
|
20
|
+
if command.buffer && command_name = global_options.render.shift
|
21
|
+
run(command_name, global_options, command.buffer, OpenStruct.new, log)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def classify(string)
|
26
|
+
string.split(/\W+/).map{|s| s.capitalize }.join("")
|
27
|
+
end
|
28
|
+
|
29
|
+
def make_manager(global_options, log)
|
30
|
+
hosts = global_options.hosts.map {|h| [h, global_options.port].join(":") }
|
31
|
+
|
32
|
+
Nameserver.new(hosts, :retries => global_options.retry,
|
33
|
+
:log => log,
|
34
|
+
:framed => global_options.framed,
|
35
|
+
:dry_run => global_options.dry)
|
17
36
|
end
|
18
|
-
end
|
19
37
|
|
20
|
-
|
21
|
-
|
38
|
+
def make_job_injector(global_options, log)
|
39
|
+
RetryProxy.new global_options.retry,
|
40
|
+
JobInjector.new(global_options.hosts.first, global_options.injector_port, log, true, global_options.dry)
|
41
|
+
end
|
22
42
|
end
|
23
43
|
|
24
|
-
attr_reader :
|
25
|
-
|
26
|
-
|
44
|
+
attr_reader :manager, :job_injector, :global_options, :argv, :command_options
|
45
|
+
|
46
|
+
def initialize(manager, job_injector, global_options, argv, command_options)
|
47
|
+
@manager = manager
|
48
|
+
@job_injector = job_injector
|
27
49
|
@global_options = global_options
|
28
50
|
@argv = argv
|
29
51
|
@command_options = command_options
|
@@ -62,49 +84,35 @@ module Gizzard
|
|
62
84
|
end
|
63
85
|
end
|
64
86
|
|
65
|
-
class
|
66
|
-
def self.make_service(global_options, log)
|
67
|
-
RetryProxy.new global_options.retry.to_i,
|
68
|
-
Gizzard::Thrift::ShardManager.new(global_options.host, global_options.port, log, global_options.framed, global_options.dry)
|
69
|
-
end
|
70
|
-
end
|
71
|
-
|
72
|
-
class JobCommand < Command
|
73
|
-
def self.make_service(global_options, log)
|
74
|
-
RetryProxy.new global_options.retry.to_i ,
|
75
|
-
Gizzard::Thrift::JobManager.new(global_options.host, global_options.port + 2, log, global_options.framed, global_options.dry)
|
76
|
-
end
|
77
|
-
end
|
78
|
-
|
79
|
-
class AddforwardingCommand < ShardCommand
|
87
|
+
class AddforwardingCommand < Command
|
80
88
|
def run
|
81
89
|
help! if argv.length != 3
|
82
90
|
table_id, base_id, shard_id_text = argv
|
83
91
|
shard_id = ShardId.parse(shard_id_text)
|
84
|
-
|
92
|
+
manager.set_forwarding(Forwarding.new(table_id.to_i, base_id.to_i, shard_id))
|
85
93
|
end
|
86
94
|
end
|
87
95
|
|
88
|
-
class DeleteforwardingCommand <
|
96
|
+
class DeleteforwardingCommand < Command
|
89
97
|
def run
|
90
98
|
help! if argv.length != 3
|
91
99
|
table_id, base_id, shard_id_text = argv
|
92
100
|
shard_id = ShardId.parse(shard_id_text)
|
93
|
-
|
101
|
+
manager.remove_forwarding(Forwarding.new(table_id.to_i, base_id.to_i, shard_id))
|
94
102
|
end
|
95
103
|
end
|
96
104
|
|
97
|
-
class HostsCommand <
|
105
|
+
class HostsCommand < Command
|
98
106
|
def run
|
99
|
-
|
107
|
+
manager.list_hostnames.map do |host|
|
100
108
|
puts host
|
101
109
|
end
|
102
110
|
end
|
103
111
|
end
|
104
112
|
|
105
|
-
class ForwardingsCommand <
|
113
|
+
class ForwardingsCommand < Command
|
106
114
|
def run
|
107
|
-
|
115
|
+
manager.get_forwardings.sort_by do |f|
|
108
116
|
[ ((f.table_id.abs << 1) + (f.table_id < 0 ? 1 : 0)), f.base_id ]
|
109
117
|
end.reject do |forwarding|
|
110
118
|
@command_options.table_ids && !@command_options.table_ids.include?(forwarding.table_id)
|
@@ -114,7 +122,7 @@ module Gizzard
|
|
114
122
|
end
|
115
123
|
end
|
116
124
|
|
117
|
-
class SubtreeCommand <
|
125
|
+
class SubtreeCommand < Command
|
118
126
|
def run
|
119
127
|
@roots = []
|
120
128
|
argv.each do |arg|
|
@@ -128,7 +136,7 @@ module Gizzard
|
|
128
136
|
end
|
129
137
|
|
130
138
|
def roots_of(id)
|
131
|
-
links =
|
139
|
+
links = manager.list_upward_links(id)
|
132
140
|
if links.empty?
|
133
141
|
[id]
|
134
142
|
else
|
@@ -137,7 +145,7 @@ module Gizzard
|
|
137
145
|
end
|
138
146
|
|
139
147
|
def down(id, depth = 0)
|
140
|
-
|
148
|
+
manager.list_downward_links(id).map do |link|
|
141
149
|
printable = " " * depth + link.down_id.to_unix
|
142
150
|
output printable
|
143
151
|
down(link.down_id, depth + 1)
|
@@ -145,21 +153,10 @@ module Gizzard
|
|
145
153
|
end
|
146
154
|
end
|
147
155
|
|
148
|
-
class ReloadCommand <
|
156
|
+
class ReloadCommand < Command
|
149
157
|
def run
|
150
158
|
if global_options.force || ask
|
151
|
-
|
152
|
-
# allow hosts to be given on the command line
|
153
|
-
@argv.each do |hostname|
|
154
|
-
output hostname
|
155
|
-
opts = global_options.dup
|
156
|
-
opts.host = hostname
|
157
|
-
s = self.class.make_service(opts, global_options.log || "./gizzmo.log")
|
158
|
-
s.reload_forwardings
|
159
|
-
end
|
160
|
-
else
|
161
|
-
service.reload_forwardings
|
162
|
-
end
|
159
|
+
manager.reload_config
|
163
160
|
else
|
164
161
|
STDERR.puts "aborted"
|
165
162
|
end
|
@@ -171,17 +168,17 @@ module Gizzard
|
|
171
168
|
end
|
172
169
|
end
|
173
170
|
|
174
|
-
class DeleteCommand <
|
171
|
+
class DeleteCommand < Command
|
175
172
|
def run
|
176
173
|
argv.each do |arg|
|
177
174
|
id = ShardId.parse(arg)
|
178
|
-
|
175
|
+
manager.delete_shard(id)
|
179
176
|
output id.to_unix
|
180
177
|
end
|
181
178
|
end
|
182
179
|
end
|
183
180
|
|
184
|
-
class AddlinkCommand <
|
181
|
+
class AddlinkCommand < Command
|
185
182
|
def run
|
186
183
|
up_id, down_id, weight = argv
|
187
184
|
help! if argv.length != 3
|
@@ -189,29 +186,29 @@ module Gizzard
|
|
189
186
|
up_id = ShardId.parse(up_id)
|
190
187
|
down_id = ShardId.parse(down_id)
|
191
188
|
link = LinkInfo.new(up_id, down_id, weight)
|
192
|
-
|
189
|
+
manager.add_link(link.up_id, link.down_id, link.weight)
|
193
190
|
output link.to_unix
|
194
191
|
end
|
195
192
|
end
|
196
193
|
|
197
|
-
class UnlinkCommand <
|
194
|
+
class UnlinkCommand < Command
|
198
195
|
def run
|
199
196
|
up_id, down_id = argv
|
200
197
|
up_id = ShardId.parse(up_id)
|
201
198
|
down_id = ShardId.parse(down_id)
|
202
|
-
|
199
|
+
manager.remove_link(up_id, down_id)
|
203
200
|
end
|
204
201
|
end
|
205
202
|
|
206
|
-
class UnwrapCommand <
|
203
|
+
class UnwrapCommand < Command
|
207
204
|
def run
|
208
205
|
shard_ids = argv
|
209
206
|
help! "No shards specified" if shard_ids.empty?
|
210
207
|
shard_ids.each do |shard_id_string|
|
211
208
|
shard_id = ShardId.parse(shard_id_string)
|
212
209
|
|
213
|
-
upward_links =
|
214
|
-
downward_links =
|
210
|
+
upward_links = manager.list_upward_links(shard_id)
|
211
|
+
downward_links = manager.list_downward_links(shard_id)
|
215
212
|
|
216
213
|
if upward_links.length == 0 or downward_links.length == 0
|
217
214
|
STDERR.puts "Shard #{shard_id_string} must not be a root or leaf"
|
@@ -220,19 +217,19 @@ module Gizzard
|
|
220
217
|
|
221
218
|
upward_links.each do |uplink|
|
222
219
|
downward_links.each do |downlink|
|
223
|
-
|
220
|
+
manager.add_link(uplink.up_id, downlink.down_id, uplink.weight)
|
224
221
|
new_link = LinkInfo.new(uplink.up_id, downlink.down_id, uplink.weight)
|
225
|
-
|
226
|
-
|
222
|
+
manager.remove_link(uplink.up_id, uplink.down_id)
|
223
|
+
manager.remove_link(downlink.up_id, downlink.down_id)
|
227
224
|
output new_link.to_unix
|
228
225
|
end
|
229
226
|
end
|
230
|
-
|
227
|
+
manager.delete_shard shard_id
|
231
228
|
end
|
232
229
|
end
|
233
230
|
end
|
234
231
|
|
235
|
-
class CreateCommand <
|
232
|
+
class CreateCommand < Command
|
236
233
|
def run
|
237
234
|
help! if argv.length < 2
|
238
235
|
class_name, *shard_ids = argv
|
@@ -241,26 +238,26 @@ module Gizzard
|
|
241
238
|
destination_type = command_options.destination_type || ""
|
242
239
|
shard_ids.each do |id|
|
243
240
|
shard_id = ShardId.parse(id)
|
244
|
-
|
245
|
-
|
241
|
+
manager.create_shard(ShardInfo.new(shard_id, class_name, source_type, destination_type, busy))
|
242
|
+
manager.get_shard(shard_id)
|
246
243
|
output shard_id.to_unix
|
247
244
|
end
|
248
245
|
end
|
249
246
|
end
|
250
247
|
|
251
|
-
class LinksCommand <
|
248
|
+
class LinksCommand < Command
|
252
249
|
def run
|
253
250
|
shard_ids = @argv
|
254
251
|
shard_ids.each do |shard_id_text|
|
255
252
|
shard_id = ShardId.parse(shard_id_text)
|
256
253
|
next if !shard_id
|
257
254
|
unless command_options.down
|
258
|
-
|
255
|
+
manager.list_upward_links(shard_id).each do |link_info|
|
259
256
|
output command_options.ids ? link_info.up_id.to_unix : link_info.to_unix
|
260
257
|
end
|
261
258
|
end
|
262
259
|
unless command_options.up
|
263
|
-
|
260
|
+
manager.list_downward_links(shard_id).each do |link_info|
|
264
261
|
output command_options.ids ? link_info.down_id.to_unix : link_info.to_unix
|
265
262
|
end
|
266
263
|
end
|
@@ -268,41 +265,41 @@ module Gizzard
|
|
268
265
|
end
|
269
266
|
end
|
270
267
|
|
271
|
-
class InfoCommand <
|
268
|
+
class InfoCommand < Command
|
272
269
|
def run
|
273
270
|
shard_ids = @argv
|
274
271
|
shard_ids.each do |shard_id|
|
275
|
-
shard_info =
|
272
|
+
shard_info = manager.get_shard(ShardId.parse(shard_id))
|
276
273
|
output shard_info.to_unix
|
277
274
|
end
|
278
275
|
end
|
279
276
|
end
|
280
277
|
|
281
|
-
class MarkbusyCommand <
|
278
|
+
class MarkbusyCommand < Command
|
282
279
|
def run
|
283
280
|
shard_ids = @argv
|
284
281
|
shard_ids.each do |shard_id|
|
285
282
|
id = ShardId.parse(shard_id)
|
286
|
-
|
287
|
-
shard_info =
|
283
|
+
manager.mark_shard_busy(id, 1)
|
284
|
+
shard_info = manager.get_shard(id)
|
288
285
|
output shard_info.to_unix
|
289
286
|
end
|
290
287
|
end
|
291
288
|
end
|
292
289
|
|
293
|
-
class MarkunbusyCommand <
|
290
|
+
class MarkunbusyCommand < Command
|
294
291
|
def run
|
295
292
|
shard_ids = @argv
|
296
293
|
shard_ids.each do |shard_id|
|
297
294
|
id = ShardId.parse(shard_id)
|
298
|
-
|
299
|
-
shard_info =
|
295
|
+
manager.mark_shard_busy(id, 0)
|
296
|
+
shard_info = manager.get_shard(id)
|
300
297
|
output shard_info.to_unix
|
301
298
|
end
|
302
299
|
end
|
303
300
|
end
|
304
301
|
|
305
|
-
class RepairCommand <
|
302
|
+
class RepairCommand < Command
|
306
303
|
def run
|
307
304
|
args = @argv.dup.map{|a| a.split(/\s+/)}.flatten
|
308
305
|
pairs = []
|
@@ -314,8 +311,8 @@ module Gizzard
|
|
314
311
|
end
|
315
312
|
pairs.each do |master, slave|
|
316
313
|
puts "#{master} #{slave}"
|
317
|
-
mprefixes =
|
318
|
-
sprefixes =
|
314
|
+
mprefixes = manager.shards_for_hostname(master).map{|s| s.id.table_prefix}
|
315
|
+
sprefixes = manager.shards_for_hostname(slave).map{|s| s.id.table_prefix}
|
319
316
|
delta = mprefixes - sprefixes
|
320
317
|
delta.each do |prefix|
|
321
318
|
puts "gizzmo copy #{master}/#{prefix} #{slave}/#{prefix}"
|
@@ -324,7 +321,7 @@ module Gizzard
|
|
324
321
|
end
|
325
322
|
end
|
326
323
|
|
327
|
-
class WrapCommand <
|
324
|
+
class WrapCommand < Command
|
328
325
|
def self.derive_wrapper_shard_id(shard_info, wrapping_class_name)
|
329
326
|
suffix = "_" + wrapping_class_name.split(".").last.downcase.gsub("shard", "")
|
330
327
|
ShardId.new("localhost", shard_info.id.table_prefix + suffix)
|
@@ -335,15 +332,15 @@ module Gizzard
|
|
335
332
|
help! "No shards specified" if shard_ids.empty?
|
336
333
|
shard_ids.each do |shard_id_string|
|
337
334
|
shard_id = ShardId.parse(shard_id_string)
|
338
|
-
shard_info =
|
339
|
-
|
335
|
+
shard_info = manager.get_shard(shard_id)
|
336
|
+
manager.create_shard(ShardInfo.new(wrapper_id = self.class.derive_wrapper_shard_id(shard_info, class_name), class_name, "", "", 0))
|
340
337
|
|
341
|
-
existing_links =
|
338
|
+
existing_links = manager.list_upward_links(shard_id)
|
342
339
|
unless existing_links.include?(LinkInfo.new(wrapper_id, shard_id, 1))
|
343
|
-
|
340
|
+
manager.add_link(wrapper_id, shard_id, 1)
|
344
341
|
existing_links.each do |link_info|
|
345
|
-
|
346
|
-
|
342
|
+
manager.add_link(link_info.up_id, wrapper_id, link_info.weight)
|
343
|
+
manager.remove_link(link_info.up_id, link_info.down_id)
|
347
344
|
end
|
348
345
|
end
|
349
346
|
output wrapper_id.to_unix
|
@@ -351,7 +348,7 @@ module Gizzard
|
|
351
348
|
end
|
352
349
|
end
|
353
350
|
|
354
|
-
class RebalanceCommand <
|
351
|
+
class RebalanceCommand < Command
|
355
352
|
|
356
353
|
class NamedArray < Array
|
357
354
|
attr_reader :name
|
@@ -413,7 +410,7 @@ module Gizzard
|
|
413
410
|
host = set.name
|
414
411
|
set.each do |id|
|
415
412
|
if id.hostname != host
|
416
|
-
shard_info ||=
|
413
|
+
shard_info ||= manager.get_shard(id)
|
417
414
|
old = id.to_unix
|
418
415
|
id.hostname = host
|
419
416
|
shards << [old, id.to_unix]
|
@@ -429,11 +426,11 @@ module Gizzard
|
|
429
426
|
end
|
430
427
|
end
|
431
428
|
|
432
|
-
class PairCommand <
|
429
|
+
class PairCommand < Command
|
433
430
|
def run
|
434
431
|
ids = []
|
435
432
|
@argv.map do |host|
|
436
|
-
|
433
|
+
manager.shards_for_hostname(host).each do |shard|
|
437
434
|
ids << shard.id
|
438
435
|
end
|
439
436
|
end
|
@@ -460,11 +457,11 @@ module Gizzard
|
|
460
457
|
displayed = {}
|
461
458
|
overlaps.sort_by { |hosts, count| count }.reverse.each do |(host_a, host_b), count|
|
462
459
|
next if !host_a || !host_b || displayed[host_a] || displayed[host_b]
|
463
|
-
id_a = ids_by_host[host_a].find {
|
464
|
-
id_b = ids_by_host[host_b].find {
|
460
|
+
id_a = ids_by_host[host_a].find {|id| manager.list_upward_links(id).size > 0 }
|
461
|
+
id_b = ids_by_host[host_b].find {|id| manager.list_upward_links(id).size > 0 }
|
465
462
|
next unless id_a && id_b
|
466
|
-
weight_a =
|
467
|
-
weight_b =
|
463
|
+
weight_a = manager.list_upward_links(id_a).first.weight
|
464
|
+
weight_b = manager.list_upward_links(id_b).first.weight
|
468
465
|
if weight_a > weight_b
|
469
466
|
puts "#{host_a}\t#{host_b}"
|
470
467
|
else
|
@@ -483,7 +480,7 @@ module Gizzard
|
|
483
480
|
end
|
484
481
|
end
|
485
482
|
|
486
|
-
class ReportCommand <
|
483
|
+
class ReportCommand < Command
|
487
484
|
def run
|
488
485
|
things = @argv.map do |shard|
|
489
486
|
parse(down(ShardId.parse(shard))).join("\n")
|
@@ -532,7 +529,7 @@ module Gizzard
|
|
532
529
|
end
|
533
530
|
|
534
531
|
def down(id)
|
535
|
-
vals =
|
532
|
+
vals = manager.list_downward_links(id).map do |link|
|
536
533
|
down(link.down_id)
|
537
534
|
end
|
538
535
|
{ id.to_unix => vals }
|
@@ -550,11 +547,11 @@ module Gizzard
|
|
550
547
|
end
|
551
548
|
end
|
552
549
|
|
553
|
-
class FindCommand <
|
550
|
+
class FindCommand < Command
|
554
551
|
def run
|
555
552
|
hosts = @argv << command_options.shard_host
|
556
553
|
hosts.compact.each do |host|
|
557
|
-
|
554
|
+
manager.shards_for_hostname(host).each do |shard|
|
558
555
|
next if command_options.shard_type && shard.class_name !~ Regexp.new(command_options.shard_type)
|
559
556
|
output shard.id.to_unix
|
560
557
|
end
|
@@ -562,7 +559,7 @@ module Gizzard
|
|
562
559
|
end
|
563
560
|
end
|
564
561
|
|
565
|
-
class LookupCommand <
|
562
|
+
class LookupCommand < Command
|
566
563
|
def run
|
567
564
|
table_id, source = @argv
|
568
565
|
help!("Requires table id and source") unless table_id && source
|
@@ -572,51 +569,51 @@ module Gizzard
|
|
572
569
|
else
|
573
570
|
source_id = source.to_i
|
574
571
|
end
|
575
|
-
shard =
|
572
|
+
shard = manager.find_current_forwarding(table_id.to_i, source_id)
|
576
573
|
output shard.id.to_unix
|
577
574
|
end
|
578
575
|
end
|
579
576
|
|
580
|
-
class CopyCommand <
|
577
|
+
class CopyCommand < Command
|
581
578
|
def run
|
582
579
|
from_shard_id_string, to_shard_id_string = @argv
|
583
580
|
help!("Requires source & destination shard id") unless from_shard_id_string && to_shard_id_string
|
584
581
|
from_shard_id = ShardId.parse(from_shard_id_string)
|
585
582
|
to_shard_id = ShardId.parse(to_shard_id_string)
|
586
|
-
|
583
|
+
manager.copy_shard(from_shard_id, to_shard_id)
|
587
584
|
end
|
588
585
|
end
|
589
586
|
|
590
|
-
class BusyCommand <
|
587
|
+
class BusyCommand < Command
|
591
588
|
def run
|
592
|
-
|
589
|
+
manager.get_busy_shards().each { |shard_info| output shard_info.to_unix }
|
593
590
|
end
|
594
591
|
end
|
595
592
|
|
596
|
-
class SetupReplicaCommand <
|
593
|
+
class SetupReplicaCommand < Command
|
597
594
|
def run
|
598
595
|
from_shard_id_string, to_shard_id_string = @argv
|
599
596
|
help!("Requires source & destination shard id") unless from_shard_id_string && to_shard_id_string
|
600
597
|
from_shard_id = ShardId.parse(from_shard_id_string)
|
601
598
|
to_shard_id = ShardId.parse(to_shard_id_string)
|
602
599
|
|
603
|
-
if
|
600
|
+
if manager.list_upward_links(to_shard_id).size > 0
|
604
601
|
STDERR.puts "Destination shard #{to_shard_id} has links to it."
|
605
602
|
exit 1
|
606
603
|
end
|
607
604
|
|
608
|
-
link =
|
605
|
+
link = manager.list_upward_links(from_shard_id)[0]
|
609
606
|
replica_shard_id = link.up_id
|
610
607
|
weight = link.weight
|
611
608
|
write_only_shard_id = ShardId.new("localhost", "#{to_shard_id.table_prefix}_copy_write_only")
|
612
|
-
|
613
|
-
|
614
|
-
|
609
|
+
manager.create_shard(ShardInfo.new(write_only_shard_id, "WriteOnlyShard", "", "", 0))
|
610
|
+
manager.add_link(replica_shard_id, write_only_shard_id, weight)
|
611
|
+
manager.add_link(write_only_shard_id, to_shard_id, 1)
|
615
612
|
output to_shard_id.to_unix
|
616
613
|
end
|
617
614
|
end
|
618
615
|
|
619
|
-
class FinishReplicaCommand <
|
616
|
+
class FinishReplicaCommand < Command
|
620
617
|
def run
|
621
618
|
from_shard_id_string, to_shard_id_string = @argv
|
622
619
|
help!("Requires source & destination shard id") unless from_shard_id_string && to_shard_id_string
|
@@ -624,58 +621,58 @@ module Gizzard
|
|
624
621
|
to_shard_id = ShardId.parse(to_shard_id_string)
|
625
622
|
|
626
623
|
write_only_shard_id = ShardId.new("localhost", "#{to_shard_id.table_prefix}_copy_write_only")
|
627
|
-
link =
|
624
|
+
link = manager.list_upward_links(write_only_shard_id)[0]
|
628
625
|
replica_shard_id = link.up_id
|
629
626
|
weight = link.weight
|
630
627
|
|
631
628
|
# careful. need to validate some basic assumptions.
|
632
629
|
unless global_options.force
|
633
|
-
if
|
630
|
+
if manager.list_upward_links(from_shard_id).map { |link| link.up_id }.to_a != [ replica_shard_id ]
|
634
631
|
STDERR.puts "Uplink from #{from_shard_id} is not a migration replica."
|
635
632
|
exit 1
|
636
633
|
end
|
637
|
-
if
|
634
|
+
if manager.list_upward_links(to_shard_id).map { |link| link.up_id }.to_a != [ write_only_shard_id ]
|
638
635
|
STDERR.puts "Uplink from #{to_shard_id} is not a write-only barrier."
|
639
636
|
exit 1
|
640
637
|
end
|
641
638
|
end
|
642
639
|
|
643
|
-
|
644
|
-
|
645
|
-
|
646
|
-
|
640
|
+
manager.remove_link(write_only_shard_id, to_shard_id)
|
641
|
+
manager.remove_link(replica_shard_id, write_only_shard_id)
|
642
|
+
manager.add_link(replica_shard_id, to_shard_id, weight)
|
643
|
+
manager.delete_shard(write_only_shard_id)
|
647
644
|
end
|
648
645
|
end
|
649
646
|
|
650
|
-
class SetupMigrateCommand <
|
647
|
+
class SetupMigrateCommand < Command
|
651
648
|
def run
|
652
649
|
from_shard_id_string, to_shard_id_string = @argv
|
653
650
|
help!("Requires source & destination shard id") unless from_shard_id_string && to_shard_id_string
|
654
651
|
from_shard_id = ShardId.parse(from_shard_id_string)
|
655
652
|
to_shard_id = ShardId.parse(to_shard_id_string)
|
656
653
|
|
657
|
-
if
|
654
|
+
if manager.list_upward_links(to_shard_id).size > 0
|
658
655
|
STDERR.puts "Destination shard #{to_shard_id} has links to it."
|
659
656
|
exit 1
|
660
657
|
end
|
661
658
|
|
662
659
|
write_only_shard_id = ShardId.new("localhost", "#{to_shard_id.table_prefix}_migrate_write_only")
|
663
660
|
replica_shard_id = ShardId.new("localhost", "#{to_shard_id.table_prefix}_migrate_replica")
|
664
|
-
|
665
|
-
|
666
|
-
|
667
|
-
|
668
|
-
|
669
|
-
|
670
|
-
end
|
671
|
-
|
672
|
-
|
673
|
-
|
661
|
+
manager.create_shard(ShardInfo.new(write_only_shard_id, "com.twitter.gizzard.shards.WriteOnlyShard", "", "", 0))
|
662
|
+
manager.create_shard(ShardInfo.new(replica_shard_id, "com.twitter.gizzard.shards.ReplicatingShard", "", "", 0))
|
663
|
+
manager.add_link(write_only_shard_id, to_shard_id, 1)
|
664
|
+
manager.list_upward_links(from_shard_id).each do |link|
|
665
|
+
manager.remove_link(link.up_id, link.down_id)
|
666
|
+
manager.add_link(link.up_id, replica_shard_id, link.weight)
|
667
|
+
end
|
668
|
+
manager.add_link(replica_shard_id, from_shard_id, 1)
|
669
|
+
manager.add_link(replica_shard_id, write_only_shard_id, 0)
|
670
|
+
manager.replace_forwarding(from_shard_id, replica_shard_id)
|
674
671
|
output replica_shard_id.to_unix
|
675
672
|
end
|
676
673
|
end
|
677
674
|
|
678
|
-
class FinishMigrateCommand <
|
675
|
+
class FinishMigrateCommand < Command
|
679
676
|
def run
|
680
677
|
from_shard_id_string, to_shard_id_string = @argv
|
681
678
|
help!("Requires source & destination shard id") unless from_shard_id_string && to_shard_id_string
|
@@ -687,57 +684,194 @@ module Gizzard
|
|
687
684
|
|
688
685
|
# careful. need to validate some basic assumptions.
|
689
686
|
unless global_options.force
|
690
|
-
if
|
687
|
+
if manager.list_upward_links(from_shard_id).map { |link| link.up_id }.to_a != [ replica_shard_id ]
|
691
688
|
STDERR.puts "Uplink from #{from_shard_id} is not a migration replica."
|
692
689
|
exit 1
|
693
690
|
end
|
694
|
-
if
|
691
|
+
if manager.list_upward_links(to_shard_id).map { |link| link.up_id }.to_a != [ write_only_shard_id ]
|
695
692
|
STDERR.puts "Uplink from #{to_shard_id} is not a write-only barrier."
|
696
693
|
exit 1
|
697
694
|
end
|
698
|
-
if
|
695
|
+
if manager.list_upward_links(write_only_shard_id).map { |link| link.up_id }.to_a != [ replica_shard_id ]
|
699
696
|
STDERR.puts "Uplink from write-only barrier is not a migration replica."
|
700
697
|
exit 1
|
701
698
|
end
|
702
699
|
end
|
703
700
|
|
704
|
-
|
705
|
-
|
706
|
-
|
707
|
-
|
701
|
+
manager.remove_link(write_only_shard_id, to_shard_id)
|
702
|
+
manager.list_upward_links(replica_shard_id).each do |link|
|
703
|
+
manager.remove_link(link.up_id, link.down_id)
|
704
|
+
manager.add_link(link.up_id, to_shard_id, link.weight)
|
708
705
|
end
|
709
|
-
|
710
|
-
|
711
|
-
|
706
|
+
manager.replace_forwarding(replica_shard_id, to_shard_id)
|
707
|
+
manager.delete_shard(replica_shard_id)
|
708
|
+
manager.delete_shard(write_only_shard_id)
|
712
709
|
end
|
713
710
|
end
|
714
711
|
|
715
|
-
class InjectCommand <
|
712
|
+
class InjectCommand < Command
|
716
713
|
def run
|
714
|
+
count = 0
|
715
|
+
page_size = 20
|
717
716
|
priority, *jobs = @argv
|
718
717
|
help!("Requires priority") unless priority and jobs.size > 0
|
719
|
-
|
720
|
-
jobs.
|
721
|
-
|
718
|
+
|
719
|
+
jobs.each_slice(page_size) do |js|
|
720
|
+
job_injector.inject_jobs(js.map {|j| Job.new(priority.to_i, j) })
|
721
|
+
|
722
722
|
count += 1
|
723
723
|
# FIXME add -q --quiet option
|
724
724
|
STDERR.print "."
|
725
|
-
STDERR.print "#{count}" if count %
|
725
|
+
STDERR.print "#{count * page_size}" if count % 10 == 0
|
726
726
|
STDERR.flush
|
727
727
|
end
|
728
728
|
STDERR.print "\n"
|
729
729
|
end
|
730
730
|
end
|
731
731
|
|
732
|
-
class FlushCommand <
|
732
|
+
class FlushCommand < Command
|
733
733
|
def run
|
734
734
|
args = @argv[0]
|
735
735
|
help!("Requires --all, or a job priority id.") unless args || command_options.flush_all
|
736
736
|
if command_options.flush_all
|
737
|
-
|
737
|
+
manager.retry_errors()
|
738
738
|
else
|
739
|
-
|
739
|
+
manager.retry_errors_for(args.to_i)
|
740
|
+
end
|
741
|
+
end
|
742
|
+
end
|
743
|
+
|
744
|
+
|
745
|
+
class AddHostCommand < Command
|
746
|
+
def run
|
747
|
+
hosts = @argv.map do |arg|
|
748
|
+
cluster, hostname, port = *arg.split(":")
|
749
|
+
help!("malformed host argument") unless [cluster, hostname, port].compact.length == 3
|
750
|
+
|
751
|
+
Host.new(hostname, port.to_i, cluster, HostStatus::Normal)
|
740
752
|
end
|
753
|
+
|
754
|
+
hosts.each {|h| manager.add_remote_host(h) }
|
755
|
+
end
|
756
|
+
end
|
757
|
+
|
758
|
+
class RemoveHostCommand < Command
|
759
|
+
def run
|
760
|
+
host = @argv[0].split(":")
|
761
|
+
host.unshift nil if host.length == 2
|
762
|
+
cluster, hostname, port = *host
|
763
|
+
|
764
|
+
manager.remove_remote_host(hostname, port.to_i)
|
765
|
+
end
|
766
|
+
end
|
767
|
+
|
768
|
+
class ListHostsCommand < Command
|
769
|
+
def run
|
770
|
+
manager.list_remote_hosts.each do |host|
|
771
|
+
puts "#{[host.cluster, host.hostname, host.port].join(":")} #{host.status}"
|
772
|
+
end
|
773
|
+
end
|
774
|
+
end
|
775
|
+
|
776
|
+
class TopologyCommand < Command
|
777
|
+
def run
|
778
|
+
templates = manager.manifest(*global_options.tables).templates.inject({}) do |h, (t, fs)|
|
779
|
+
h.update t.to_config => fs
|
780
|
+
end
|
781
|
+
|
782
|
+
if command_options.forwardings
|
783
|
+
templates.
|
784
|
+
inject([]) { |h, (t, fs)| fs.each { |f| h << [f.base_id, t] }; h }.
|
785
|
+
sort.
|
786
|
+
each { |a| puts "%25d\t%s" % a }
|
787
|
+
else
|
788
|
+
templates.
|
789
|
+
map { |(t, fs)| [fs.length, t] }.
|
790
|
+
sort.reverse.
|
791
|
+
each { |a| puts "%4d %s" % a }
|
792
|
+
end
|
793
|
+
end
|
794
|
+
end
|
795
|
+
|
796
|
+
class TransformTreeCommand < Command
|
797
|
+
def run
|
798
|
+
help!("wrong number of arguments") unless @argv.length == 2
|
799
|
+
|
800
|
+
scheduler_options = command_options.scheduler_options || {}
|
801
|
+
template_s, shard_id_s = @argv
|
802
|
+
|
803
|
+
to_template = ShardTemplate.parse(template_s)
|
804
|
+
shard_id = ShardId.parse(shard_id_s)
|
805
|
+
base_name = shard_id.table_prefix.split('_').first
|
806
|
+
forwarding = manager.get_forwarding_for_shard(shard_id)
|
807
|
+
manifest = manager.manifest(forwarding.table_id)
|
808
|
+
shard = manifest.trees[forwarding]
|
809
|
+
copy_wrapper = scheduler_options[:copy_wrapper]
|
810
|
+
be_quiet = global_options.force && command_options.quiet
|
811
|
+
transformation = Transformation.new(shard.template, to_template, copy_wrapper)
|
812
|
+
|
813
|
+
scheduler_options[:quiet] = be_quiet
|
814
|
+
|
815
|
+
unless be_quiet
|
816
|
+
puts transformation.inspect
|
817
|
+
puts ""
|
818
|
+
end
|
819
|
+
|
820
|
+
unless global_options.force
|
821
|
+
print "Continue? (y/n) "; $stdout.flush
|
822
|
+
exit unless $stdin.gets.chomp == "y"
|
823
|
+
puts ""
|
824
|
+
end
|
825
|
+
|
826
|
+
Gizzard.schedule! manager,
|
827
|
+
base_name,
|
828
|
+
{ transformation => { forwarding => shard } },
|
829
|
+
scheduler_options
|
830
|
+
end
|
831
|
+
end
|
832
|
+
|
833
|
+
class TransformCommand < Command
|
834
|
+
def run
|
835
|
+
help!("must have an even number of arguments") unless @argv.length % 2 == 0
|
836
|
+
|
837
|
+
scheduler_options = command_options.scheduler_options || {}
|
838
|
+
manifest = manager.manifest(*global_options.tables)
|
839
|
+
copy_wrapper = scheduler_options[:copy_wrapper]
|
840
|
+
be_quiet = global_options.force && command_options.quiet
|
841
|
+
transformations = {}
|
842
|
+
|
843
|
+
scheduler_options[:quiet] = be_quiet
|
844
|
+
|
845
|
+
@argv.each_slice(2) do |(from_template_s, to_template_s)|
|
846
|
+
from, to = [from_template_s, to_template_s].map {|s| ShardTemplate.parse(s) }
|
847
|
+
transformation = Transformation.new(from, to, copy_wrapper)
|
848
|
+
forwardings = Set.new(manifest.templates[from] || [])
|
849
|
+
trees = manifest.trees.reject {|(f, s)| !forwardings.include?(f) }
|
850
|
+
|
851
|
+
transformations[transformation] = trees
|
852
|
+
end
|
853
|
+
|
854
|
+
base_name = transformations.values.first.values.first.id.table_prefix.split('_').first
|
855
|
+
|
856
|
+
unless be_quiet
|
857
|
+
transformations.each do |transformation, trees|
|
858
|
+
puts transformation.inspect
|
859
|
+
puts "Applied to #{trees.length} shards:"
|
860
|
+
trees.keys.sort.each {|f| puts " #{f.inspect}" }
|
861
|
+
end
|
862
|
+
puts ""
|
863
|
+
end
|
864
|
+
|
865
|
+
unless global_options.force
|
866
|
+
print "Continue? (y/n) "; $stdout.flush
|
867
|
+
exit unless $stdin.gets.chomp == "y"
|
868
|
+
puts ""
|
869
|
+
end
|
870
|
+
|
871
|
+
Gizzard.schedule! manager,
|
872
|
+
base_name,
|
873
|
+
transformations,
|
874
|
+
scheduler_options
|
741
875
|
end
|
742
876
|
end
|
743
877
|
end
|