braavos 0.0.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,30 @@
1
+ require 'tempfile'
2
+
3
+ class Braavos::Command
4
+ class CommandError < StandardError; end
5
+
6
+ NUMBER_OF_TRIES = 3
7
+
8
+ def execute(script, input)
9
+ script_tf = Tempfile.new('brav-cmd')
10
+ script_tf.write(script)
11
+ script_tf.close
12
+ FileUtils.chmod("+rx", script_tf.path)
13
+
14
+ execute_with_retry script, script_tf, input
15
+ ensure
16
+ script_tf.unlink
17
+ end
18
+
19
+ private
20
+
21
+ def execute_with_retry(script, script_tf, input)
22
+ tries ||= 0
23
+ Braavos.logger.debug("Attempt #{tries + 1} - Command Execute: #{script} [#{input.join(' ')}]")
24
+ system(script_tf.path, *input) || raise(CommandError, "Command execution failed: #{script}")
25
+ rescue CommandError
26
+ tries += 1
27
+ retry if tries < NUMBER_OF_TRIES
28
+ raise
29
+ end
30
+ end
@@ -0,0 +1,48 @@
1
+ class Braavos::Config
2
+
3
+ attr_accessor :settings, :name, :environment, :service, :bucket_name, :data_loc, :sync_loc, :temp_loc,
4
+ :discovery, :backup_prefix, :storage_backing, :parallel_jobs
5
+
6
+ def initialize(config)
7
+ @settings = config
8
+ config.each do |k,v|
9
+ send("#{k}=", v) if respond_to?("#{k}=")
10
+ end
11
+ self.backup_prefix = nil if backup_prefix && backup_prefix.strip.size == 0
12
+ self.storage_backing ||= 's3'
13
+ self.temp_loc ||= ''
14
+
15
+ FileUtils.mkdir_p temp_loc unless File.directory?(temp_loc)
16
+ raise "Missing temp location: #{temp_loc}" unless File.directory?(temp_loc)
17
+ end
18
+
19
+ def service_class
20
+ @service_class ||= -> do
21
+ Object.const_get("Braavos::Service::#{service.capitalize}")
22
+ end.call
23
+ end
24
+
25
+ def storage_class
26
+ @storage_class ||= -> do
27
+ Object.const_get("Braavos::Storage::#{storage_backing.capitalize}")
28
+ end.call
29
+ end
30
+
31
+ def backup_path(reset=false)
32
+ remove_instance_variable(:@backup_path) if reset
33
+ @backup_path ||= -> do
34
+ path = [backup_prefix, name, environment, service].compact
35
+ File.join(*path)
36
+ end.call
37
+ end
38
+
39
+ def get_regex(key)
40
+ if (regex = settings[key]) && regex.size > 2
41
+ if regex[0] == regex[-1] && regex[0] == '/'
42
+ regex = regex[1...-1]
43
+ end
44
+ /#{regex}/
45
+ end
46
+ end
47
+
48
+ end
@@ -0,0 +1,104 @@
1
+ require 'tempfile'
2
+
3
+ class Braavos::Parallel
4
+
5
+ attr_accessor :parallel_cmd, :parallel_jobs
6
+
7
+ #wget http://ftp.gnu.org/gnu/parallel/parallel-latest.tar.bz2
8
+ #tar -xvjf parallel*
9
+ #cd parallel*
10
+ #./configure && make && sudo make install
11
+
12
+ def initialize(command = "parallel", parallel_jobs = "100%")
13
+ @parallel_cmd = command
14
+ @parallel_jobs = parallel_jobs
15
+
16
+ validate_parallel_installed
17
+ validate_timeout_installed
18
+ end
19
+
20
+ def execute(script, input, options={})
21
+ script_temp_file = Tempfile.new('brav-parl-scr')
22
+ script_temp_file.write(script)
23
+ script_temp_file.close
24
+ FileUtils.chmod("+rx", script_temp_file.path)
25
+
26
+ input_files = generate_input_files(input)
27
+
28
+ begin
29
+ input_file_names = input_files.map{|inf| inf.path}.join(' ')
30
+ command = generate_command(script_temp_file, input_file_names, options)
31
+ Braavos.logger.info("Parallel Execute: #{command}")
32
+
33
+ rval = execute_command command
34
+
35
+ unless rval
36
+ Braavos.logger.info("Retrying failed parallel jobs.")
37
+ retry_command = generate_command(script_temp_file, input_file_names, options, true)
38
+ rval = execute_command retry_command
39
+ end
40
+
41
+ raise StandardError, "Failed to run backup due to errored jobs" unless rval
42
+ ensure
43
+ input_files.each do |inf| inf.close! end
44
+ end
45
+ end
46
+
47
+ def generate_input_files(input)
48
+ input_files = []
49
+
50
+ if (first = input.first) && first.is_a?(Array)
51
+ (0..first.size).each do |i|
52
+ input_tf = Tempfile.new('brav-parl-inp')
53
+ input_files << input_tf
54
+ File.open(input_tf.path, 'w') do |f|
55
+ input.each do |minput|
56
+ f.puts minput[i]
57
+ end
58
+ end
59
+ end
60
+ else
61
+ input_tf = Tempfile.new('parl-inp')
62
+ input_files << input_tf
63
+ File.open(input_tf.path, 'w') do |f|
64
+ input.each do |minput|
65
+ f.puts minput
66
+ end
67
+ end
68
+ end
69
+
70
+ input_files
71
+ end
72
+
73
+ def generate_command(script_temp_file, input_file_names, options, resume_failed = false)
74
+ joblog_path = "/tmp/braavos_joblog.txt"
75
+ if resume_failed
76
+ "parallel --no-notice --joblog #{joblog_path} --resume-failed -j #{parallel_jobs} #{options[:parallel_opts] || ''} --xapply #{script_temp_file.path} :::: #{input_file_names} "
77
+ else
78
+ "parallel --no-notice --joblog #{joblog_path} -j #{parallel_jobs} #{options[:parallel_opts] || ''} --xapply #{script_temp_file.path} :::: #{input_file_names} "
79
+ end
80
+ end
81
+
82
+ def execute_command(command)
83
+ system(command)
84
+ success = $?.exitstatus == 0
85
+ Braavos.logger.warn("Parallel execution failed with status #{$?.exitstatus}") unless success
86
+ success
87
+ end
88
+
89
+ def validate_parallel_installed
90
+ `#{parallel_cmd} --version`
91
+ rescue StandardError
92
+ raise ArgumentError, "GNU parallel not available"
93
+ end
94
+
95
+ def validate_timeout_installed
96
+ `timeout --version`
97
+ rescue StandardError
98
+ begin
99
+ `gtimeout --version`
100
+ rescue StandardError
101
+ raise ArgumentError, "timeout from GNU coreutils is not available"
102
+ end
103
+ end
104
+ end
@@ -0,0 +1,48 @@
1
+ module Braavos::Service
2
+ autoload :Cassandra, 'braavos/service/cassandra'
3
+ autoload :Elasticsearch, 'braavos/service/elasticsearch'
4
+
5
+ class << self
6
+
7
+ def instance_id
8
+ @instance_id ||= -> {
9
+ instance_id = case Braavos.config.discovery
10
+ when 'ec2'
11
+ `curl -s http://169.254.169.254/latest/meta-data/instance-id`.chomp.downcase
12
+ else
13
+ `hostname -s`.chomp.downcase
14
+ end
15
+ Braavos.logger.info "Instance ID: #{instance_id}"
16
+ instance_id
17
+ }.call
18
+ end
19
+
20
+ def public_host_ip
21
+ @public_host_ip ||= -> {
22
+ public_host_ip = case Braavos.config.discovery
23
+ when 'ec2'
24
+ `curl -s http://169.254.169.254/latest/meta-data/public-ipv4`.chomp.downcase
25
+ else
26
+ "127.0.0.1"
27
+ end
28
+ public_host_ip
29
+ }.call
30
+ end
31
+
32
+ def local_host_ip
33
+ @local_host_ip ||= -> {
34
+ local_host_ip = case Braavos.config.discovery
35
+ when 'ec2'
36
+ `curl -s http://169.254.169.254/latest/meta-data/local-ipv4`.chomp.downcase
37
+ else
38
+ "127.0.0.1"
39
+ end
40
+ local_host_ip
41
+ }.call
42
+ end
43
+
44
+ def full_backup_id(time=Time.now, format="%Y%m%d-%H%M%S")
45
+ time.strftime(format)
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,187 @@
1
+ require 'tmpdir'
2
+
3
+ class Braavos::Service::Cassandra
4
+
5
+ DEFAULT_KEYSPACES_SYSTEM = ['system', 'system_auth', 'system_traces']
6
+
7
+ def token
8
+ @token ||= safely_retrieve_token
9
+ end
10
+
11
+ def keyspaces
12
+ @keyspaces ||= -> do
13
+ result = Dir[File.join(Braavos.config.data_loc, '*')].map do |d|
14
+ d.sub(/\A#{Braavos.config.data_loc}\//, '')
15
+ end
16
+ Braavos.logger.debug("Found cassandra keyspaces: #{result}")
17
+ result
18
+ end.call
19
+ end
20
+
21
+ def keyspaces_data
22
+ @keyspaces_data ||= -> do
23
+ if regex = Braavos.config.get_regex('cassandra_keyspaces_data')
24
+ Braavos.logger.debug("Cassandra Data Keyspaces: regex = #{regex}")
25
+ keyspaces.select do |k| regex =~ k end
26
+ else
27
+ keyspaces - DEFAULT_KEYSPACES_SYSTEM
28
+ end
29
+ end.call
30
+ end
31
+
32
+ def keyspaces_system
33
+ @keyspaces_system ||= -> do
34
+ if regex = Braavos.config.get_regex('cassandra_keyspaces_system')
35
+ Braavos.logger.debug("Cassandra System Keyspaces: regex = #{regex}")
36
+ keyspaces.select do |k| regex =~ k end
37
+ else
38
+ DEFAULT_KEYSPACES_SYSTEM & keyspaces
39
+ end
40
+ end.call
41
+ end
42
+
43
+ def backup_full
44
+ backup_id = Braavos::Service.full_backup_id
45
+ backup_path = File.join(Braavos.config.backup_path, 'full', backup_id, Braavos.storage.find_node_id)
46
+ data_path = File.join(Braavos.config.backup_path, 'data', Braavos.storage.find_node_id)
47
+
48
+ if Braavos.storage.has_success?(backup_path)
49
+ raise "Backup currently exists: #{backup_id} - #{Braavos.storage.script_path(backup_path)}"
50
+ else
51
+ Braavos.storage.clear_result(backup_path)
52
+ end
53
+
54
+ Dir.mktmpdir('brav-bkup') do |tmpd|
55
+ File.write(File.join(tmpd, 'cluster.json'), JSON.pretty_generate(Braavos.storage.get_cluster))
56
+
57
+ write_whoami(tmpd)
58
+ keyspaces_data.each do |keyspace|
59
+ write_describering(tmpd, keyspace)
60
+ end
61
+
62
+ Braavos.command.execute(Braavos.template.load_template('cassandra/system_bundle.sh.erb'), [tmpd, Braavos.config.data_loc, *keyspaces_system])
63
+
64
+ Braavos.parallel.execute(Braavos.template.load_template('cassandra/dump_schema.sh.erb', local_host_ip: Braavos::Service.local_host_ip), keyspaces_data.map{|k| [k, tmpd]})
65
+
66
+ Dir["#{tmpd}/*"].each do |f|
67
+ Braavos.storage.write_file(File.join(backup_path, File.basename(f)), file: f)
68
+ end
69
+ end
70
+
71
+ script = Braavos.template.load_template('cassandra/create_snapshot.sh.erb')
72
+ keyspaces_data.each do |keyspace|
73
+ Braavos.command.execute(script, [keyspace, backup_id])
74
+ end
75
+
76
+ table_list = list_tables("snapshots/#{backup_id}")
77
+
78
+ contents = Hash[tables: table_list.map do |t| "#{t[1]}.tgz" end]
79
+ Braavos.storage.write_file(File.join(backup_path, 'contents.json'), JSON.pretty_generate(contents))
80
+
81
+ # remove tables that are unchanged
82
+ data_listings = Braavos.storage.list_dir(data_path).map do |k, v|
83
+ k.sub(/^#{data_path}\//, '').sub(/\.tgz\Z/, '')
84
+ end
85
+ table_list.delete_if do |t|
86
+ data_listings.include? t[1]
87
+ end
88
+ Braavos.logger.debug("executing table_list: #{table_list}")
89
+ Braavos.logger.info("Processing Table Count: #{table_list.size}")
90
+
91
+ script = Braavos.template.load_template('cassandra/table_bundle_upload.sh.erb')
92
+ script_input = table_list.map do |t|
93
+ [t[0], Braavos.storage.script_path(File.join(data_path, "#{t[1]}.tgz"))]
94
+ end
95
+ Braavos.parallel.execute(script, script_input) if script_input.size > 0
96
+
97
+ Braavos.storage.write_file(File.join(backup_path, '_COMPLETED'), '')
98
+ rescue => e
99
+ begin
100
+ Braavos.storage.write_file(File.join(backup_path, '_FAILED'), "#{e.message}\n#{e.backtrace}")
101
+ rescue => ig
102
+ Braavos.logger.error("_FAILED failed, ignoring")
103
+ end
104
+ raise e
105
+ ensure
106
+ script = Braavos.template.load_template('cassandra/clear_snapshot.sh.erb')
107
+ keyspaces_data.each do |keyspace|
108
+ begin
109
+ Braavos.command.execute(script, [keyspace, backup_id])
110
+ rescue => e
111
+ Braavos.logger.error("clear snapshot #{keyspace} failed, ignoring")
112
+ end
113
+ end
114
+ end
115
+
116
+ def restore(backup_loc, restore_loc)
117
+ backup_path = File.join(Braavos.config.backup_path, backup_loc, Braavos.storage.find_node_id)
118
+ data_path = File.join(Braavos.config.backup_path, 'data', Braavos.storage.find_node_id)
119
+
120
+ Braavos.storage.load_file(File.join(backup_path, '_COMPLETED'))
121
+
122
+ contents = JSON.parse(Braavos.storage.load_file(File.join(backup_path, 'contents.json')))
123
+
124
+ script = Braavos.template.load_template('cassandra/table_bundle_restore.sh.erb')
125
+ script_input = contents['tables'].map do |t|
126
+ [Braavos.storage.script_path(File.join(data_path, t)), File.join(restore_loc, File.dirname(t))]
127
+ end
128
+ Braavos.parallel.execute(script, script_input) if script_input.size > 0
129
+
130
+ Braavos.logger.info("Restore completed: #{backup_loc} to #{restore_loc}")
131
+ end
132
+
133
+ def restore_incr
134
+
135
+ end
136
+
137
+ # return [[disk_loc, 'table_name']]
138
+ def list_tables(location)
139
+ # Example for table_name:
140
+ # /usr/local/var/lib/cassandra/data/place_directory_development/Places/snapshots/TODAY/place_directory_development-Places-ib-1-CompressionInfo.db
141
+ # returned as "place_directory_development/Places/place_directory_development-Places-ib-1"
142
+ tables = Set.new
143
+ Dir[File.join(Braavos.config.data_loc, '**', location, '*')].each do |file|
144
+ next if file =~ /.json\Z/ # Some versions of cassandra use Table and Table.index json files in the sstable storage location
145
+ file.sub!(/-[\w.]+\Z/, '')
146
+ if match = file.match(/\A#{Braavos.config.data_loc}\/([\w\/]+)\/#{location}\/([-\w\.]+)\Z/)
147
+ ks_table, ssfile = match.captures
148
+ tables << [file, File.join(ks_table, ssfile)]
149
+ else
150
+ Braavos.logger.warn("Found unexpected file in snapshot: #{file}")
151
+ end
152
+ end
153
+ tables.to_a
154
+ end
155
+
156
+ def find_snapshots
157
+ results = `find #{Braavos.config.data_loc} -type d -wholename '*/snapshots/*'`.split("\n").select {|s| not s =~ /\/_.*$/ }
158
+
159
+ end
160
+
161
+ private
162
+
163
+ def write_describering(directory, keyspace)
164
+ path = File.join(directory, "#{keyspace}_describering.txt")
165
+ File.open(path, "w") do |f|
166
+ f.puts `nodetool describering #{keyspace}`
167
+ end
168
+ end
169
+
170
+ def write_whoami(directory)
171
+ path = File.join(directory, "whoami.txt")
172
+ File.open(path, "w") do |f|
173
+ f.puts "instance_id:#{Braavos::Service.instance_id}"
174
+ f.puts "local_host_ip:#{Braavos::Service.local_host_ip}"
175
+ f.puts "public_host_ip:#{Braavos::Service.public_host_ip}"
176
+ f.puts "token:#{token}"
177
+ end
178
+ end
179
+
180
+ def safely_retrieve_token
181
+ tries ||= 2
182
+ `nodetool info | head -1 | cut -d : -f 2 | sed -e 's/^[ \t]*//'`
183
+ rescue StandardError
184
+ retry unless (tries -= 1).zero?
185
+ raise
186
+ end
187
+ end
@@ -0,0 +1,15 @@
1
+ class Braavos::Service::Elasticsearch
2
+
3
+ def initialize
4
+
5
+ end
6
+
7
+ def backup_full
8
+
9
+ end
10
+
11
+ def restore_full
12
+
13
+ end
14
+
15
+ end
@@ -0,0 +1,7 @@
1
+ module Braavos::Storage
2
+
3
+ autoload :StorageBase, 'braavos/storage/storage_base'
4
+ autoload :S3, 'braavos/storage/s3'
5
+ autoload :File, 'braavos/storage/file'
6
+
7
+ end