braavos 0.0.7

data/lib/braavos/command.rb

@@ -0,0 +1,30 @@
+require 'tempfile'
+
+class Braavos::Command
+  class CommandError < StandardError; end
+
+  NUMBER_OF_TRIES = 3
+
+  def execute(script, input)
+    script_tf = Tempfile.new('brav-cmd')
+    script_tf.write(script)
+    script_tf.close
+    FileUtils.chmod("+rx", script_tf.path)
+
+    execute_with_retry script, script_tf, input
+  ensure
+    script_tf.unlink
+  end
+
+  private
+
+  def execute_with_retry(script, script_tf, input)
+    tries ||= 0
+    Braavos.logger.debug("Attempt #{tries + 1} - Command Execute: #{script} [#{input.join(' ')}]")
+    system(script_tf.path, *input) || raise(CommandError, "Command execution failed: #{script}")
+  rescue CommandError
+    tries += 1
+    retry if tries < NUMBER_OF_TRIES
+    raise
+  end
+end

data/lib/braavos/config.rb

@@ -0,0 +1,48 @@
+class Braavos::Config
+
+  attr_accessor :settings, :name, :environment, :service, :bucket_name, :data_loc, :sync_loc, :temp_loc,
+                :discovery, :backup_prefix, :storage_backing, :parallel_jobs
+
+  def initialize(config)
+    @settings = config
+    config.each do |k,v|
+      send("#{k}=", v) if respond_to?("#{k}=")
+    end
+    self.backup_prefix = nil if backup_prefix && backup_prefix.strip.size == 0
+    self.storage_backing ||= 's3'
+    self.temp_loc ||= ''
+
+    FileUtils.mkdir_p temp_loc unless File.directory?(temp_loc)
+    raise "Missing temp location: #{temp_loc}" unless File.directory?(temp_loc)
+  end
+
+  def service_class
+    @service_class ||= -> do
+      Object.const_get("Braavos::Service::#{service.capitalize}")
+    end.call
+  end
+
+  def storage_class
+    @storage_class ||= -> do
+      Object.const_get("Braavos::Storage::#{storage_backing.capitalize}")
+    end.call
+  end
+
+  def backup_path(reset=false)
+    remove_instance_variable(:@backup_path) if reset
+    @backup_path ||= -> do
+      path = [backup_prefix, name, environment, service].compact
+      File.join(*path)
+    end.call
+  end
+
+  def get_regex(key)
+    if (regex = settings[key]) && regex.size > 2
+      if regex[0] == regex[-1] && regex[0] == '/'
+        regex = regex[1...-1]
+      end
+      /#{regex}/
+    end
+  end
+
+end

data/lib/braavos/parallel.rb

@@ -0,0 +1,104 @@
+require 'tempfile'
+
+class Braavos::Parallel
+
+  attr_accessor :parallel_cmd, :parallel_jobs
+
+  #wget http://ftp.gnu.org/gnu/parallel/parallel-latest.tar.bz2
+  #tar -xvjf parallel*
+  #cd parallel*
+  #./configure && make && sudo make install
+
+  def initialize(command = "parallel", parallel_jobs = "100%")
+    @parallel_cmd = command
+    @parallel_jobs = parallel_jobs
+
+    validate_parallel_installed
+    validate_timeout_installed
+  end
+
+  def execute(script, input, options={})
+    script_temp_file = Tempfile.new('brav-parl-scr')
+    script_temp_file.write(script)
+    script_temp_file.close
+    FileUtils.chmod("+rx", script_temp_file.path)
+
+    input_files = generate_input_files(input)
+
+    begin
+      input_file_names = input_files.map{|inf| inf.path}.join(' ')
+      command = generate_command(script_temp_file, input_file_names, options)
+      Braavos.logger.info("Parallel Execute: #{command}")
+
+      rval = execute_command command
+
+      unless rval
+        Braavos.logger.info("Retrying failed parallel jobs.")
+        retry_command = generate_command(script_temp_file, input_file_names, options, true)
+        rval = execute_command retry_command
+      end
+
+      raise StandardError, "Failed to run backup due to errored jobs" unless rval
+    ensure
+      input_files.each do |inf| inf.close! end
+    end
+  end
+
+  def generate_input_files(input)
+    input_files = []
+
+    if (first = input.first) && first.is_a?(Array)
+      (0..first.size).each do |i|
+        input_tf = Tempfile.new('brav-parl-inp')
+        input_files << input_tf
+        File.open(input_tf.path, 'w') do |f|
+          input.each do |minput|
+            f.puts minput[i]
+          end
+        end
+      end
+    else
+      input_tf = Tempfile.new('parl-inp')
+      input_files << input_tf
+      File.open(input_tf.path, 'w') do |f|
+        input.each do |minput|
+          f.puts minput
+        end
+      end
+    end
+
+    input_files
+  end
+
+  def generate_command(script_temp_file, input_file_names, options, resume_failed = false)
+    joblog_path = "/tmp/braavos_joblog.txt"
+    if resume_failed
+      "parallel --no-notice --joblog #{joblog_path} --resume-failed -j #{parallel_jobs} #{options[:parallel_opts] || ''} --xapply #{script_temp_file.path} :::: #{input_file_names} "
+    else
+      "parallel --no-notice --joblog #{joblog_path} -j #{parallel_jobs} #{options[:parallel_opts] || ''} --xapply #{script_temp_file.path} :::: #{input_file_names} "
+    end
+  end
+
+  def execute_command(command)
+    system(command)
+    success = $?.exitstatus == 0
+    Braavos.logger.warn("Parallel execution failed with status #{$?.exitstatus}") unless success
+    success
+  end
+
+  def validate_parallel_installed
+    `#{parallel_cmd} --version`
+  rescue StandardError
+    raise ArgumentError, "GNU parallel not available"
+  end
+
+  def validate_timeout_installed
+    `timeout --version`
+  rescue StandardError
+    begin
+      `gtimeout --version`
+    rescue StandardError
+      raise ArgumentError, "timeout from GNU coreutils is not available"
+    end
+  end
+end

data/lib/braavos/service.rb

@@ -0,0 +1,48 @@
+module Braavos::Service
+  autoload :Cassandra, 'braavos/service/cassandra'
+  autoload :Elasticsearch, 'braavos/service/elasticsearch'
+
+  class << self
+
+    def instance_id
+      @instance_id ||= -> {
+        instance_id = case Braavos.config.discovery
+          when 'ec2'
+            `curl -s http://169.254.169.254/latest/meta-data/instance-id`.chomp.downcase
+          else
+            `hostname -s`.chomp.downcase
+        end
+        Braavos.logger.info "Instance ID: #{instance_id}"
+        instance_id
+      }.call
+    end
+
+    def public_host_ip
+      @public_host_ip ||= -> {
+        public_host_ip = case Braavos.config.discovery
+          when 'ec2'
+            `curl -s http://169.254.169.254/latest/meta-data/public-ipv4`.chomp.downcase
+          else
+            "127.0.0.1"
+        end
+        public_host_ip
+      }.call
+    end
+
+    def local_host_ip
+      @local_host_ip ||= -> {
+        local_host_ip = case Braavos.config.discovery
+          when 'ec2'
+            `curl -s http://169.254.169.254/latest/meta-data/local-ipv4`.chomp.downcase
+          else
+            "127.0.0.1"
+        end
+        local_host_ip
+      }.call
+    end
+
+    def full_backup_id(time=Time.now, format="%Y%m%d-%H%M%S")
+      time.strftime(format)
+    end
+  end
+end

data/lib/braavos/service/cassandra.rb

@@ -0,0 +1,187 @@
+require 'tmpdir'
+
+class Braavos::Service::Cassandra
+
+  DEFAULT_KEYSPACES_SYSTEM = ['system', 'system_auth', 'system_traces']
+
+  def token
+    @token ||= safely_retrieve_token
+  end
+
+  def keyspaces
+    @keyspaces ||= -> do
+      result = Dir[File.join(Braavos.config.data_loc, '*')].map do |d|
+        d.sub(/\A#{Braavos.config.data_loc}\//, '')
+      end
+      Braavos.logger.debug("Found cassandra keyspaces: #{result}")
+      result
+    end.call
+  end
+
+  def keyspaces_data
+    @keyspaces_data ||= -> do
+      if regex = Braavos.config.get_regex('cassandra_keyspaces_data')
+        Braavos.logger.debug("Cassandra Data Keyspaces: regex = #{regex}")
+        keyspaces.select do |k| regex =~ k end
+      else
+        keyspaces - DEFAULT_KEYSPACES_SYSTEM
+      end
+    end.call
+  end
+
+  def keyspaces_system
+    @keyspaces_system ||= -> do
+      if regex = Braavos.config.get_regex('cassandra_keyspaces_system')
+        Braavos.logger.debug("Cassandra System Keyspaces: regex = #{regex}")
+        keyspaces.select do |k| regex =~ k end
+      else
+        DEFAULT_KEYSPACES_SYSTEM & keyspaces
+      end
+    end.call
+  end
+
+  def backup_full
+    backup_id = Braavos::Service.full_backup_id
+    backup_path = File.join(Braavos.config.backup_path, 'full', backup_id, Braavos.storage.find_node_id)
+    data_path = File.join(Braavos.config.backup_path, 'data', Braavos.storage.find_node_id)
+
+    if Braavos.storage.has_success?(backup_path)
+      raise "Backup currently exists: #{backup_id} - #{Braavos.storage.script_path(backup_path)}"
+    else
+      Braavos.storage.clear_result(backup_path)
+    end
+
+    Dir.mktmpdir('brav-bkup') do |tmpd|
+      File.write(File.join(tmpd, 'cluster.json'), JSON.pretty_generate(Braavos.storage.get_cluster))
+
+      write_whoami(tmpd)
+      keyspaces_data.each do |keyspace|
+        write_describering(tmpd, keyspace)
+      end
+
+      Braavos.command.execute(Braavos.template.load_template('cassandra/system_bundle.sh.erb'), [tmpd, Braavos.config.data_loc, *keyspaces_system])
+
+      Braavos.parallel.execute(Braavos.template.load_template('cassandra/dump_schema.sh.erb', local_host_ip: Braavos::Service.local_host_ip), keyspaces_data.map{|k| [k, tmpd]})
+
+      Dir["#{tmpd}/*"].each do |f|
+        Braavos.storage.write_file(File.join(backup_path, File.basename(f)), file: f)
+      end
+    end
+
+    script = Braavos.template.load_template('cassandra/create_snapshot.sh.erb')
+    keyspaces_data.each do |keyspace|
+      Braavos.command.execute(script, [keyspace, backup_id])
+    end
+
+    table_list = list_tables("snapshots/#{backup_id}")
+
+    contents = Hash[tables: table_list.map do |t| "#{t[1]}.tgz" end]
+    Braavos.storage.write_file(File.join(backup_path, 'contents.json'), JSON.pretty_generate(contents))
+
+    # remove tables that are unchanged
+    data_listings = Braavos.storage.list_dir(data_path).map do |k, v|
+      k.sub(/^#{data_path}\//, '').sub(/\.tgz\Z/, '')
+    end
+    table_list.delete_if do |t|
+      data_listings.include? t[1]
+    end
+    Braavos.logger.debug("executing table_list: #{table_list}")
+    Braavos.logger.info("Processing Table Count: #{table_list.size}")
+
+    script = Braavos.template.load_template('cassandra/table_bundle_upload.sh.erb')
+    script_input = table_list.map do |t|
+      [t[0], Braavos.storage.script_path(File.join(data_path, "#{t[1]}.tgz"))]
+    end
+    Braavos.parallel.execute(script, script_input) if script_input.size > 0
+
+    Braavos.storage.write_file(File.join(backup_path, '_COMPLETED'), '')
+  rescue => e
+    begin
+      Braavos.storage.write_file(File.join(backup_path, '_FAILED'), "#{e.message}\n#{e.backtrace}")
+    rescue => ig
+      Braavos.logger.error("_FAILED failed, ignoring")
+    end
+    raise e
+  ensure
+    script = Braavos.template.load_template('cassandra/clear_snapshot.sh.erb')
+    keyspaces_data.each do |keyspace|
+      begin
+        Braavos.command.execute(script, [keyspace, backup_id])
+      rescue => e
+        Braavos.logger.error("clear snapshot #{keyspace} failed, ignoring")
+      end
+    end
+  end
+
+  def restore(backup_loc, restore_loc)
+    backup_path = File.join(Braavos.config.backup_path, backup_loc, Braavos.storage.find_node_id)
+    data_path = File.join(Braavos.config.backup_path, 'data', Braavos.storage.find_node_id)
+
+    Braavos.storage.load_file(File.join(backup_path, '_COMPLETED'))
+
+    contents = JSON.parse(Braavos.storage.load_file(File.join(backup_path, 'contents.json')))
+
+    script = Braavos.template.load_template('cassandra/table_bundle_restore.sh.erb')
+    script_input = contents['tables'].map do |t|
+      [Braavos.storage.script_path(File.join(data_path, t)), File.join(restore_loc, File.dirname(t))]
+    end
+    Braavos.parallel.execute(script, script_input) if script_input.size > 0
+
+    Braavos.logger.info("Restore completed: #{backup_loc} to #{restore_loc}")
+  end
+
+  def restore_incr
+
+  end
+
+  # return [[disk_loc, 'table_name']]
+  def list_tables(location)
+    # Example for table_name:
+    # /usr/local/var/lib/cassandra/data/place_directory_development/Places/snapshots/TODAY/place_directory_development-Places-ib-1-CompressionInfo.db
+    # returned as "place_directory_development/Places/place_directory_development-Places-ib-1"
+    tables = Set.new
+    Dir[File.join(Braavos.config.data_loc, '**', location, '*')].each do |file|
+      next if file =~ /.json\Z/ # Some versions of cassandra use Table and Table.index json files in the sstable storage location
+      file.sub!(/-[\w.]+\Z/, '')
+      if match = file.match(/\A#{Braavos.config.data_loc}\/([\w\/]+)\/#{location}\/([-\w\.]+)\Z/)
+        ks_table, ssfile = match.captures
+        tables << [file, File.join(ks_table, ssfile)]
+      else
+        Braavos.logger.warn("Found unexpected file in snapshot: #{file}")
+      end
+    end
+    tables.to_a
+  end
+
+  def find_snapshots
+    results = `find #{Braavos.config.data_loc} -type d -wholename '*/snapshots/*'`.split("\n").select {|s| not s =~ /\/_.*$/ }
+
+  end
+
+  private
+
+  def write_describering(directory, keyspace)
+    path = File.join(directory, "#{keyspace}_describering.txt")
+    File.open(path, "w") do |f|
+      f.puts `nodetool describering #{keyspace}`
+    end
+  end
+
+  def write_whoami(directory)
+    path = File.join(directory, "whoami.txt")
+    File.open(path, "w") do |f|
+      f.puts "instance_id:#{Braavos::Service.instance_id}"
+      f.puts "local_host_ip:#{Braavos::Service.local_host_ip}"
+      f.puts "public_host_ip:#{Braavos::Service.public_host_ip}"
+      f.puts "token:#{token}"
+    end
+  end
+
+  def safely_retrieve_token
+    tries ||= 2
+    `nodetool info | head -1 | cut -d : -f 2 | sed -e 's/^[ \t]*//'`
+  rescue StandardError
+    retry unless (tries -= 1).zero?
+    raise
+  end
+end

data/lib/braavos/service/elasticsearch.rb

@@ -0,0 +1,15 @@
+class Braavos::Service::Elasticsearch
+
+  def initialize
+
+  end
+
+  def backup_full
+
+  end
+
+  def restore_full
+
+  end
+
+end

data/lib/braavos/storage.rb

@@ -0,0 +1,7 @@
+module Braavos::Storage
+
+  autoload :StorageBase, 'braavos/storage/storage_base'
+  autoload :S3, 'braavos/storage/s3'
+  autoload :File, 'braavos/storage/file'
+
+end