backarch 0.1.15

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 39179cbea17666cbbb5a280fbe00e8cddc62a2c4
+  data.tar.gz: 2f11255acfb638caa374316974e9d62075ba5dab
+SHA512:
+  metadata.gz: ae56c4ec655a418e0486574aa38fdc51d5a009f7eef74aa5c62cf8fb096c33783256d870d6f7de0db63d1b697f68f64e7ddfb62a76ea9fbbb0a35047f9fc8f48
+  data.tar.gz: 79616e3cf7d6a2e753f4f65e73967ef5799d8b8227b28e329687abc918ad3f91bbfb101a1966feba38e463222aacd8aab35ee2b50350d832e6052a96608c3e98

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in backarch.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Chris DiMartino
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,84 @@
+# Backarch
+
+Ability to backup and restore ElasticSearch to AWS S3.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'backarch'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install backarch
+
+## Usage
+
+Common usage of backarch involves passing the following:
+1. configuration file (-c)
+2. input (-i)
+3. output (-o)
+4. command (eg: elasticsearch)
+5. subcommand (eg: archive)
+
+##### Example:
+
+```
+snapshot_archive -c samples/elasticsearch_archive.yaml -i /tmp/elasticsearch_snapshots/full -o /tmp/elasticsearch_snapshots/archive elasticsearch archive
+```
+
+Further examples can be found in the samples/ folder.
+
+### Backarch examples (see samples/ folder)
+
+#### Commands below will backup and then restore elasticsearch from s3
+
+1. ./samples/elasticsearch_snapshot.sh # Create local snapshot
+2. ./samples/elasticsearch_archive.sh # Push snapshot to s3
+3. ./samples/elasticsearch_restore.sh
+
+#### Backup
+
+This consists of the follow two steps
+
+1. ./samples/elasticsearch_snapshot.sh # Create local snapshot
+2. ./samples/elasticsearch_archive.sh # Push snapshot to s3
+
+##### Snapshot
+
+```
+DATE=`date +%Y%m%d`
+snapshot_archive -c tmp/elasticsearch_archive.yaml -i /usr/local/var/elasticsearch/ -o /tmp/elasticsearch_snapshots -l debug -d $DATE elasticsearch snapshot
+```
+
+##### Archive
+
+```
+DATE=`date +%Y%m%d`
+snapshot_archive -c samples/elasticsearch_archive.yaml -i /tmp/elasticsearch_snapshots/full -o /tmp/elasticsearch_snapshots/archive -l debug -d $DATE elasticsearch archive
+```
+
+#### Restore
+
+Restoration involves downloading a multivolume tar file from s3 and then extracting the contents:
+
+3. ./samples/elasticsearch_restore.sh
+
+##### Shell
+
+```
+DATE=`date +%Y%m%d`
+snapshot_archive -c tmp/elasticsearch_archive.yaml -i s3://backups.qa.data-axle.infogroup.com/development/elasticsearch/full/place_directory/20140606/nil/20140606 -o /tmp/elasticsearch_restoration -l debug elasticsearch restore
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/backarch.gemspec

@@ -0,0 +1,32 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'backarch/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "backarch"
+  spec.version       = Backarch::VERSION
+  spec.authors       = ["Chris DiMartino"]
+  spec.email         = ["chris.dimartino@infogroup.com"]
+  spec.description   = %q{Backup and archival utility for cassandra/elasticsearch}
+  spec.summary       = %q{Backup and archival to cloud}
+  spec.homepage      = ""
+  spec.license       = ""
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+
+  %w{
+    logger
+    slop
+    aws-sdk
+    fork
+    fog
+    gli
+  }.map { |gem| spec.add_dependency gem }
+end

data/bin/snapshot_archive

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require 'backarch'
+exit Backarch.run

data/lib/backarch.rb

@@ -0,0 +1,40 @@
+require 'logger'
+
+require 'backarch/archive'
+require 'backarch/config'
+require 'backarch/snapshot'
+require 'backarch/snapshot/cassandra'
+require 'backarch/snapshot/elasticsearch'
+require 'backarch/version'
+require 'backarch/parallel_downloader'
+require 'backarch/tar_wrapper'
+require 'backarch/tar_extracter'
+require 'backarch/elasticsearch_restoration'
+
+class String
+  def pass
+    yield self
+  end
+end
+
+module Backarch
+  LOG = Logger.new STDERR
+  LOG.level = Logger::INFO
+
+  NODE_NAME = `ec2metadata --instance-id`.chomp rescue 'nil'
+  ADDRESS   = `ec2metadata --local-ipv4`.chomp rescue 'nil'
+
+  LOG_LEVELS = {
+    'fatal' => ::Logger::FATAL,
+    'error' => ::Logger::ERROR,
+    'warn'  => ::Logger::WARN,
+    'info'  => ::Logger::INFO,
+    'debug' => ::Logger::DEBUG
+  }
+
+  class << self
+    def run
+      Config.init
+    end
+  end
+end

data/lib/backarch/archive.rb

@@ -0,0 +1,148 @@
+require 'fork'
+require 'fog'
+require 'aws-sdk'
+require 'fileutils'
+require 'tmpdir'
+
+module Backarch
+  class Archive
+    class << self
+      def run
+        o = new
+        o.create
+        Config.config.each_pair do |provider, params|
+          o.push_archive_to provider, params
+        end
+        o.delete Config.config.keys
+      end
+    end
+
+    def create
+      if local_archive_complete?
+        LOG.info "Archive of snapshot #{archive_name} already completed"
+        return
+      end
+
+      LOG.info "Creating archive of snapshot: #{archive_name}"
+      TarWrapper.create_multiple_volumes(snapshot_folder, archive_folder)
+      local_archive_complete!
+    end
+
+    def delete providers
+      if local_archive_complete? and not providers.select { |provider| remote_archive_complete? provider }.empty? and Dir.exists? archive_folder
+        LOG.info "Removing archive folder #{archive_folder}"
+        FileUtils.rm_rf archive_folder
+        FileUtils.rm_rf snapshot_folder
+      end
+    end
+
+    def push_archive_to provider, archive_info
+      if provider == "google"
+        raise ArgumentError, "Google no longer supported"
+      end
+
+      $0 = "#{Config.program_name} archive pushing archive to #{provider}"
+      if remote_archive_complete? provider
+        LOG.warn "Remote archival of snapshot #{archive_name} already completed"
+        return
+      end
+
+      unless local_archive_complete?
+        LOG.error "Snapshot #{archive_name} not locally present"
+        return
+      end
+
+      begin
+        remote_path = "#{archive_info["path"]}#{remote_archive_path(archive_info)}"
+        start_time = Time.now
+        LOG.info("Beginning archive of #{snapshot_folder} to #{provider}://#{archive_info["bucket"]}/#{remote_path} @ #{Time.now}")
+
+        send("push_to_#{provider}".to_sym, archive_info, remote_path, archive_folder)
+        end_time = Time.now
+        LOG.info("Completed archive of #{snapshot_folder} to #{provider} @ #{end_time} - #{end_time - start_time}s elapsed")
+      rescue Exception => er
+        LOG.error("Failed to push archive #{snapshot_folder} to #{provider} @ #{Time.now}: #{er.message}")
+        er.backtrace.each { |line| LOG.error line }
+      end
+    end
+
+    private
+
+    def snapshot_folder
+      File.join Config.input, Config.date
+    end
+
+    def archive_folder
+      File.join Config.output, Config.date
+    end
+
+    def archive_name
+      "#{Config.date}"
+    end
+
+    def local_archive_complete?
+      File.exists? local_archive_complete_file
+    end
+
+    def local_archive_complete!
+      ::FileUtils.touch local_archive_complete_file
+    end
+
+    def local_archive_complete_file
+      "#{Config.output}/.#{archive_name}.local.done"
+    end
+
+    def remote_archive_complete? provider
+      File.exists? remote_archive_complete_file_for(provider)
+    end
+
+    def remote_archive_complete! provider
+      ::FileUtils.rm remote_archive_in_progress_file_for(provider) rescue nil
+      ::FileUtils.touch remote_archive_complete_file_for(provider)
+    end
+
+    def remote_archive_complete_file_for provider
+      "#{Config.output}/.#{archive_name}.remote.#{provider}.done"
+    end
+
+    def push_to_aws(provider, path, folder)
+      bucket_name = provider["bucket"]
+      if remote_archive_complete? "aws"
+        LOG.info "Remote archive has already been completed."
+        return
+      end
+
+      s3 = AWS::S3.new
+      s3_bucket = s3.buckets[bucket_name]
+      raise "Bucket does not exist" unless s3_bucket.exists?
+
+      command = "s3cmd put -r -v --multipart-chunk-size-mb=512 #{folder} s3://#{bucket_name}/#{path}"
+      LOG.info "Invoking: #{command}"
+      LOG.debug `#{command}`
+
+      if $?.to_i == 0
+        push_completed_marker_to_aws(bucket_name, path)
+        remote_archive_complete! "aws"
+      else
+        raise StandardError, "Unable to push archive to aws"
+      end
+    end
+
+    def remote_archive_path opts
+      "/%s%s/%s/" % [
+                                  opts[:path],
+                                  Config.date,
+                                  NODE_NAME
+                                ]
+    end
+
+    def push_completed_marker_to_aws(bucket, path)
+      Dir.mktmpdir do |dir|
+        files = FileUtils.touch(File.join(dir, "_COMPLETED"))
+        command = "s3cmd put -v #{files.first} s3://#{bucket}/#{File.join(path, "_COMPLETED")}"
+        LOG.info "Pushing completed marker: #{command}"
+        raise StandardError, "Unable to push _COMPLETED marker" unless system(command)
+      end
+    end
+  end
+end

data/lib/backarch/config.rb

@@ -0,0 +1,148 @@
+require 'gli'
+
+module Backarch
+  class Config
+    class << self
+      include GLI::App
+
+      attr_accessor :program_name
+      attr_accessor :target_app
+      attr_accessor :opts
+
+      DATE_FORMAT = '%Y%m%d'
+
+      def init
+        @program_name = $0
+
+        program_desc 'Snapshot and archival suite for Cassandra and ElasticSearch'
+        version Backarch::VERSION
+
+        desc 'The Cassandra, ElasticSearch, or tar data directory'
+        flag :i, :input
+
+        desc "Configuration file location (default: /etc/chef/elasticsearch_archive.yaml)"
+        flag :c, :config
+
+        desc "The date for the archive (defaults to today)"
+        flag :d, :date
+
+        desc "Log level (info, debug...)"
+        default_value 'info'
+        flag :l, :log_level
+
+        desc "The snapshot or tar output location"
+        flag :o, :output
+
+        desc "Prior snapshot directory to hard link to with rsync"
+        flag :p, :link_prior
+
+        desc 'Disable compression of archive'
+        switch :Z, :no_compress
+
+        desc 'Full path to the nodetool binary'
+        flag(:n, :nodetool)
+
+        desc 'The version of the cassandra application (required for pre 1.0 versions)'
+        flag(:V, :cassandra_version)
+
+        command(:cassandra) do |cmd|
+          cmd.action do |global_opts, opts, args|
+            @target_app = :cassandra
+            init_aws
+
+            case args.first
+            when 'snapshot'
+              Snapshot::Cassandra.run
+            when 'archive'
+              Archive.run
+            end
+          end
+        end
+
+        command(:elasticsearch) do |cmd|
+          cmd.action do |global_opts, opts, args|
+            @target_app = :elasticsearch
+            init_aws
+
+            case args.first
+            when 'snapshot'
+              Snapshot::Elasticsearch.run
+            when 'archive'
+              Archive.run
+            when 'restore'
+              ElasticsearchRestoration.run
+            end
+          end
+        end
+
+        command(:tar) do |cmd|
+          cmd.action do |global_opts, opts, args|
+            @target_app = :tar
+
+            case args.first
+            when 'extract'
+              TarExtracter.run
+            end
+          end
+        end
+
+        pre do |global_opts, command, opts, args|
+          @opts = global_opts.merge(opts)
+          LOG.level = LOG_LEVELS[@opts[:log_level]]
+        end
+
+        run(ARGV)
+      end
+
+      def init_aws
+        AWS.config \
+          :access_key_id     => config['aws']["aws_access_key_id"],
+          :secret_access_key => config['aws']["aws_secret_access_key"],
+          :ssl_verify_peer   => false,
+          :http_open_timeout => 2,
+          :http_read_timeout => 2,
+          :http_idle_timeout => 5
+      end
+
+      def compress
+        @opts[:no_compress] ? false : true
+      end
+
+      def config
+        @opts[:config] ||= case @target_app
+                           when :elasticsearch then '/etc/chef/elasticsearch_archive.yaml'
+                           when :cassandra then '/etc/chef/cassandra_archive.yaml'
+                           end
+        YAML.load(File.read(@opts[:config]))
+      end
+
+      def date
+        @opts[:date] ||= Date.today.strftime('%Y%m%d')
+      end
+
+      def cassandra_version
+        @opts[:cassandra_version]
+      end
+
+      def output
+        @opts[:output].gsub(%r{/\Z}, '') if @opts[:output]
+      end
+
+      def input
+        @opts[:input].gsub(%r{/\Z}, '') if @opts[:input]
+      end
+
+      def nodetool
+        @opts[:nodetool] ||= '/usr/lib/cassandra/bin/nodetool'
+      end
+
+      def link_prior
+        @opts[:link_prior] ||= (1..30).map { |i| snapshot_dir(i * -1) }.select { |dir| Dir.exists? dir }.first
+      end
+
+      def snapshot_dir offset=0
+        "#{Config.output}/full/#{(Date.strptime(date, DATE_FORMAT) + offset).strftime(DATE_FORMAT)}"
+      end
+    end
+  end
+end

data/lib/backarch/elasticsearch_restoration.rb

@@ -0,0 +1,41 @@
+require 'tmpdir'
+
+module Backarch
+  class ElasticsearchRestoration
+    def self.run
+      instance = new
+      instance.run
+    end
+
+    def run
+      download
+      devolume
+      LOG.info "Restoration completed. Contents in #{output}"
+    end
+
+    private
+
+    def download
+      ParallelDownloader.download_folder input, tmpdir
+    end
+
+    def devolume
+      TarWrapper.restore_from_volumes tmpdir, output
+    end
+
+    def input
+      raise ArgumentError, "Requires input parameter to be passed." unless Config.input
+      raise ArgumentError, "Input must be an s3 path (eg: s3://bucket)" unless Config.input.start_with? "s3://"
+      Config.input
+    end
+
+    def tmpdir
+      @tmpdir ||= Dir.mktmpdir
+    end
+
+    def output
+      raise ArgumentError, "Requires output parameter to be passed." unless Config.output
+      Config.output
+    end
+  end
+end

data/lib/backarch/parallel_downloader.rb

@@ -0,0 +1,67 @@
+require 'tempfile'
+
+module Backarch
+  class ParallelDownloader
+    attr_reader :s3_inputs, :destination, :outputs
+
+    def self.download_folder(s3_path, destination)
+      s3_inputs = retrieve_volumes_from_s3(s3_path)
+      instance = new destination, s3_inputs
+      instance.download
+    end
+
+    def initialize(destination, s3_inputs)
+      validate_parallel_installed
+
+      @s3_inputs = s3_inputs
+      @destination = destination
+      @outputs = generate_output_files(destination, s3_inputs)
+    end
+
+    def download
+      LOG.info "Downloading: #{s3_inputs}"
+      tuples = s3_inputs.zip(outputs)
+      file = Tempfile.new('parallel_input')
+      tuples.each do |pair|
+        file.puts "s3cmd get -v #{pair[0]} #{pair[1]}"
+      end
+      file.close
+
+      command = "parallel --no-notice -j 70% --joblog #{joblog_path} -a #{file.path}"
+      LOG.info "Running in parallel: #{command}"
+      if !system(command)
+        LOG.info "Parallel downloads failed. Retrying."
+        raise StandardError, "Failed to complete parallel download" unless system(command + " --resume-failed")
+      end
+    ensure
+      file.unlink
+    end
+
+    private
+
+    def self.retrieve_volumes_from_s3(full_path)
+      s3 = AWS::S3.new
+      bucket_name = Config.config["aws"]["bucket"]
+      bucket = s3.buckets[bucket_name]
+      path = URI.parse(full_path).path[1..-1]
+      files = bucket.objects.with_prefix(File.join(path, "volume")).map(&:key)
+      files.map { |file| File.join("s3://", bucket_name, file) }
+    end
+
+    def validate_parallel_installed
+      `parallel --version`
+    rescue StandardError
+      raise ArgumentError, "GNU parallel not available"
+    end
+
+    def joblog_path
+      "/tmp/backarch_parallel_joblog.txt"
+    end
+
+    def generate_output_files(destination, s3_inputs)
+      s3_inputs.map do |s3_input|
+        File.join(destination, File.basename(s3_input))
+      end
+    end
+  end
+end

data/lib/backarch/snapshot.rb

@@ -0,0 +1,23 @@
+module Backarch
+  class Snapshot
+    def snapshot_name
+      "full_#{Config.date}"
+    end
+
+    class << self
+      def run
+        o = self.new
+        begin
+          o.request
+          o.sync
+        ensure
+          o.delete
+        end
+      end
+    end
+
+    def snapshot_destination
+      Config.snapshot_dir
+    end
+  end
+end

data/lib/backarch/snapshot/cassandra.rb

@@ -0,0 +1,48 @@
+module Backarch
+  class Snapshot
+    class Cassandra < Snapshot
+      def request
+        LOG.info "Requesting snapshot #{snapshot_name}"
+        # Create a new snapshot in cassandra with the requested name
+        case Config.cassandra_version
+        when /\A0\.6\./
+          `#{Config.nodetool} snapshot #{snapshot_name}`
+          `find #{Config.input} -type f -wholename '*/backups/*' -exec rm {} \\; 2>/dev/null`
+          ## Going to move the directory it created to what we expect it to be now.
+          `for snapshot in \`find #{Config.input} -type d -wholename '*/snapshots/*-#{snapshot_name}'\`; do basedir=\`dirname $snapshot\`; rm -rfv $basedir/#{snapshot_name}; mv $snapshot $basedir/#{snapshot_name}; done`
+        else
+          `#{Config.nodetool} snapshot -t #{snapshot_name} 2>/dev/null`
+          `find #{Config.input} -type f -wholename '*/backups/*' -exec rm {} \\;`
+        end
+      end
+
+      def sync
+        $0 = "#{Config.program_name} snapshotter syncing #{snapshot_name}"
+        LOG.info 'Ready to sync snapshot'
+        snapshot_dirs = `find #{Config.input} -type d -wholename '*/snapshots/#{snapshot_name}'`.split(/\n/)
+        snapshot_dirs.map do |dir|
+          dest_dir = dir[%r{/data/(.+)/snapshots/}, 1]
+          prior_dir = Config.link_prior
+          link_prior = prior_dir ? "--link-dest=#{prior_dir}/#{dest_dir}" : ''
+          command = "rsync -ar --stats #{link_prior} #{dir}/ #{snapshot_destination}/#{dest_dir}"
+          LOG.debug command
+
+          FileUtils.mkdir_p "#{snapshot_destination}/#{dest_dir}"
+          FileUtils.chmod 0700, "#{snapshot_destination}/#{dest_dir}"
+          FileUtils.chown  'root', 'root', "#{snapshot_destination}/#{dest_dir}"
+          `#{command}`
+        end
+      end
+
+      def delete
+        LOG.info("Removing snapshot #{snapshot_name}")
+        case Config.cassandra_version
+        when /\A0\.6\./
+          `find #{Config.input} -type d -wholename '*/snapshots/#{snapshot_name}' -exec rm -rf {} \\;`
+        else
+          `#{Config.nodetool} clearsnapshot -t #{snapshot_name}`
+        end
+      end
+    end
+  end
+end

data/lib/backarch/snapshot/elasticsearch.rb

@@ -0,0 +1,99 @@
+require 'fileutils'
+
+module Backarch
+  class Snapshot
+    class Elasticsearch < Snapshot
+      def request
+        snapshot_destination.pass do |dir|
+          ::FileUtils.mkdir_p dir
+          ::FileUtils.chmod 0700, dir
+          ::FileUtils.chown 'root', 'root', dir
+        end
+
+        depth = Config.input.split('/').count
+
+        File.open("#{snapshot_destination}/cluster_state.local.json", "w") { |f| f.write `curl -s XGET "http://localhost:9200/_cluster/state?pretty=true&local=true"` }
+        File.open("#{snapshot_destination}/cluster_state.global.json", "w") { |f| f.write `curl -s XGET "http://localhost:9200/_cluster/state?pretty=true"` }
+
+        # now lets tar up our data files. these are huge, so lets be nice
+        indexes = `find #{Config.input} -mindepth #{depth} -maxdepth #{depth} -type d -wholename '*/indices/*' | sort`.split
+        indexes.map do |idx|
+          index_name = idx.split('/').last
+
+          LOG.info("Beginning snapshot of index #{index_name}")
+
+          es_index_flush(index_name) do
+            settings = `curl -s -XGET "http://localhost:9200/#{index_name}/_settings?pretty=true" 2>/dev/null | sed '1,2d' | sed '$d' | sed '$d'`.chomp
+            mappings = `curl -s -XGET "http://localhost:9200/#{index_name}/_mapping?pretty=true" 2>/dev/null | sed '1,2d' | sed '$d' | sed '$d'`.chomp
+
+  # time to create our restore script! oh god scripts creating scripts, this never ends well…
+            File.open("#{snapshot_destination}/#{index_name}.restore.sh", 'w') { |file| file.write(<<EOF) }
+#!/bin/bash
+# this script requires #{index_name}.tar.gz and will restore it into elasticsearch
+# it is ESSENTIAL that the index you are restoring does NOT exist in ES. delete it
+# if it does BEFORE trying to restore data.
+
+curl -s -XPUT 'http://localhost:9200/#{index_name}' -d '{#{settings}},"mappings":{#{mappings}}'
+echo
+
+# extract our data files into place
+echo "Restoring index (this may take a while)..."
+tar xvf -C #{Config.input} --strip-components=#{depth} #{index_name}.tar.gz
+echo
+EOF
+
+            command = "nice -n 19 tar czf #{snapshot_destination}/#{index_name}.tar.gz -C #{Config.input} #{idx}"
+            LOG.info("Creating archive: #{command}")
+            `#{command}`
+          end
+        end
+      end
+
+      def sync
+        # NOOP
+      end
+
+      def delete
+        # NOOP
+      end
+
+      def es_index_flush index
+        begin
+          AWS::SimpleDB.consistent_reads do
+            begin
+              sdb = AWS::SimpleDB.new
+              domain = sdb.domains.create(Config.config["aws"]["path"].gsub('/','-'))
+              lock = domain.items["#{index}-lock"]
+
+              LOG.info("Current lock before flush: #{lock.attributes['owner'].values}")
+              unless lock.attributes['owner'].values.count > 0 ## Lock already taken
+                LOG.info "Flushing index #{index}"
+                `curl -s -XPOST "http://localhost:9200/#{index}/_flush"`
+
+                LOG.info "Disabling flush of transaction logs for #{index}"
+                `curl -s -XPUT "http://localhost:9200/#{index}/_settings?pretty=true" -d '{"index":{"translog":{"disable_flush":"true"}}}'`
+              end
+              lock.attributes['owner'].add(NODE_NAME)
+              LOG.info("Current lock after flush: #{lock.attributes['owner'].values}")
+
+              yield
+            ensure
+              lock.attributes['owner'].delete(NODE_NAME) rescue nil
+              owners = lock.attributes['owner'].values rescue []
+              LOG.info("Current lock before renable flush: #{owners}")
+
+              unless owners.count > 0 ## Lock still in use. Dont enable flush yet
+                LOG.info "Enabling flush of transaction logs for #{index}"
+                `curl -s -XPUT "http://localhost:9200/#{index}/_settings?pretty=true" -d '{"index":{"translog":{"disable_flush":"false"}}}'`
+                lock.delete if lock
+              end
+            end
+          end
+        rescue AWS::SimpleDB::Errors::ServiceUnavailable
+          LOG.info "Caught AWS::SimpleDB::Errors::ServiceUnavailable error.  Retrying in 5 seconds"
+          retry
+        end
+      end
+    end
+  end
+end

data/lib/backarch/tar_extracter.rb

@@ -0,0 +1,27 @@
+require 'backarch/tar_wrapper'
+
+module Backarch
+  class TarExtracter
+    def self.run
+      instance = new
+      instance.run
+    end
+
+    def run
+      LOG.info "Restoring volumes in #{input} to #{output}"
+      TarWrapper.restore_from_volumes(input, output)
+    end
+
+    private
+
+    def input
+      raise ArgumentError, "Requires input parameter to be passed." unless Config.input
+      Config.input
+    end
+
+    def output
+      raise ArgumentError, "Requires output parameter to be passed." unless Config.output
+      Config.output
+    end
+  end
+end

data/lib/backarch/tar_wrapper.rb

@@ -0,0 +1,69 @@
+require 'fileutils'
+
+module Backarch
+  class TarWrapper
+    #BLOCKSIZE = 10737418 # 10MB
+    BLOCKSIZE = 1_073_741_824 # 1GB
+
+    class << self
+      # Invokes GNU tar to generate a multi-volume tar for parallel upload and download.
+      # eg: gtar -cML 30720 -f file1.tar -f file2.tar -C /tmp/elasticsearch_snapshots/full 20140605
+      def create_multiple_volumes(origin, destination)
+        FileUtils.mkdir_p destination
+
+        tar = validate_gnu_tar_installed
+        output_files = generate_output_files(origin, destination)
+        tape_limit = BLOCKSIZE / 1024
+        working_directory = File.dirname origin
+        basename = File.basename origin
+
+        command = "#{tar} -cML #{tape_limit} -f #{output_files.join(" -f ")} -C #{working_directory} #{basename}"
+        LOG.info "Creating tar volumes: #{command}"
+        return StandardError, "Failed to create tar volumes" unless system(command)
+      end
+
+      def restore_from_volumes(origin, destination)
+        FileUtils.mkdir_p destination
+
+        tar = validate_gnu_tar_installed
+        input_files = generate_input_files(origin)
+        command = "#{tar} -xMf #{input_files.join(" -f ")} -C #{destination}"
+        LOG.info "Restoring from tar volumes: #{command}"
+        return StandardError, "Failed to restore from tar volumes" unless system(command)
+      end
+
+      private
+
+      def generate_input_files(folder)
+        Dir.glob(File.join(folder, "**", "volume*.tar"))
+      end
+
+      def generate_output_files(origin, destination)
+        volumes = number_of_volumes(origin)
+        raise StandardError, "Incompatible number of volumes detected from folder. Does snapshot folder #{origin} exist?" unless volumes > 0
+
+        files = []
+        volumes.times { |n| files << File.join(destination, "volume#{n}.tar") }
+        files
+      end
+
+      def number_of_volumes(folder)
+        command = "BLOCKSIZE=#{BLOCKSIZE} du -d 1 #{folder} | cut -f 1"
+        `#{command}`.to_i
+      end
+
+      def validate_gnu_tar_installed
+        raise StandardError unless `tar --version`.include?("GNU")
+        "tar"
+      rescue StandardError
+        begin
+          `gtar --version`
+        rescue StandardError
+          raise ArgumentError, "GNU tar is not available. This differs to OSX's tar."
+        else
+          "gtar"
+        end
+      end
+    end
+  end
+end

data/lib/backarch/version.rb

@@ -0,0 +1,3 @@
+module Backarch
+  VERSION = "0.1.15"
+end

data/samples/elasticsearch_archive.sh

@@ -0,0 +1,3 @@
+DATE=`date +%Y%m%d`
+ruby -I.:lib ./bin/snapshot_archive -c samples/elasticsearch_archive.yaml -i /tmp/elasticsearch_snapshots/full -o /tmp/elasticsearch_snapshots/archive -l debug -d $DATE elasticsearch archive
+

data/samples/elasticsearch_archive.yaml

@@ -0,0 +1,6 @@
+---
+aws:
+  aws_access_key_id: MYACCESSKEY
+  aws_secret_access_key: MYSECRET
+  bucket: backups.qa.data-axle.infogroup.com
+  path: development/elasticsearch/full/place_directory

data/samples/elasticsearch_devolume.sh

@@ -0,0 +1,3 @@
+DATE=`date +%Y%m%d`
+ruby -I.:lib ./bin/snapshot_archive -i /tmp/elasticsearch_snapshots/archive/$DATE -o /tmp/elasticsearch_snapshots/devolume tar extract
+

data/samples/elasticsearch_restore.sh

@@ -0,0 +1,3 @@
+DATE=`date +%Y%m%d`
+ruby -I.:lib ./bin/snapshot_archive -c tmp/elasticsearch_archive.yaml -i s3://backups.qa.data-axle.infogroup.com/development/elasticsearch/full/place_directory/20140606/nil/20140606 -o /tmp/elasticsearch_restoration -l debug elasticsearch restore
+

data/samples/elasticsearch_snapshot.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+DATE=`date +%Y%m%d`
+ruby -I.:lib ./bin/snapshot_archive -c tmp/elasticsearch_archive.yaml -i /usr/local/var/elasticsearch/ -o /tmp/elasticsearch_snapshots -l debug -d $DATE elasticsearch snapshot

metadata

@@ -0,0 +1,180 @@
+--- !ruby/object:Gem::Specification
+name: backarch
+version: !ruby/object:Gem::Version
+  version: 0.1.15
+platform: ruby
+authors:
+- Chris DiMartino
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-06-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: slop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fork
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fog
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: gli
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Backup and archival utility for cassandra/elasticsearch
+email:
+- chris.dimartino@infogroup.com
+executables:
+- snapshot_archive
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- backarch.gemspec
+- bin/snapshot_archive
+- lib/backarch.rb
+- lib/backarch/archive.rb
+- lib/backarch/config.rb
+- lib/backarch/elasticsearch_restoration.rb
+- lib/backarch/parallel_downloader.rb
+- lib/backarch/snapshot.rb
+- lib/backarch/snapshot/cassandra.rb
+- lib/backarch/snapshot/elasticsearch.rb
+- lib/backarch/tar_extracter.rb
+- lib/backarch/tar_wrapper.rb
+- lib/backarch/version.rb
+- samples/elasticsearch_archive.sh
+- samples/elasticsearch_archive.yaml
+- samples/elasticsearch_devolume.sh
+- samples/elasticsearch_restore.sh
+- samples/elasticsearch_snapshot.sh
+homepage: ''
+licenses:
+- ''
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.1
+signing_key: 
+specification_version: 4
+summary: Backup and archival to cloud
+test_files: []