braavos 0.0.7

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f48996bb89970d302bac43a156684ad867b2fa13
+  data.tar.gz: 82b44ccb9e8ddc0fea2b01a0ad47a1f45a3d1ce6
+SHA512:
+  metadata.gz: 04e073c3cce5c0c7dd0d0a25b26f9a114162e6bce51688f1ed2e20e95bfbdb6cb10c86082ffa43c1af2dd27c184a81250f76ce2d931c2ef0b0cfcb654ed961c8
+  data.tar.gz: 161846c8ee93cb51f3ea55f8b8088e945a636ebb16e51a7234b30dccf5bfacba429c7480608ad8d86daaee1068a58c75d79863306abe686d43030e5a1c148d10

data/Gemfile

@@ -0,0 +1,12 @@
+source "http://rubygems.org"
+
+gem 'gli'
+gem 'aws-sdk'
+gem 'rake'
+
+group :test do
+  gem 'pry'
+  gem 'fakeweb'
+  gem 'simplecov', require: false
+  gem 'sqlite3'
+end

data/Gemfile.lock

@@ -0,0 +1,44 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    aws-sdk (1.33.0)
+      json (~> 1.4)
+      nokogiri (>= 1.4.4)
+      uuidtools (~> 2.1)
+    coderay (1.1.0)
+    docile (1.1.0)
+    fakeweb (1.3.0)
+    gli (2.9.0)
+    json (1.8.1)
+    lockfile (2.1.0)
+    method_source (0.8.2)
+    mini_portile (0.5.3)
+    multi_json (1.8.4)
+    nokogiri (1.6.1)
+      mini_portile (~> 0.5.0)
+    pry (0.9.12.6)
+      coderay (~> 1.0)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+    rake (10.1.1)
+    simplecov (0.8.1)
+      docile (~> 1.1.0)
+      lockfile (>= 2.1.0)
+      multi_json
+      simplecov-html (~> 0.8.0)
+    simplecov-html (0.8.0)
+    slop (3.5.0)
+    sqlite3 (1.3.8)
+    uuidtools (2.1.4)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  aws-sdk
+  fakeweb
+  gli
+  pry
+  rake
+  simplecov
+  sqlite3

data/README.rdoc

@@ -0,0 +1,113 @@
+= braavos - Backup/Restore Tool
+
+== Introduction
+
+This document covers the implementation of a backup/restore utility named braavos and deployed on Cassandra and ElasticSearch servers. 
+It implements:
+
+* the management of backup data on S3
+  * Backup operations
+  * Restore operations
+  * Pruning executions
+* full backup strategies (IMPLEMENTED)
+* restore capabilities
+  * rewind to previous state
+  * restore fallen node (IMPLEMENTED)
+  * restore from different cluster (IMPLEMENTED)
+* quick replacement strategy (using ebs volume)
+* secondary backup strategy (google drive)
+
+=== Versions Targeted:
+* Cassandra: 1.2
+* ElasticSearch: 0.90
+
+== Terminology
+
+Service : Cassandra(cass) or ElasticSearch(es)
+
+Node : A running instance of a service.
+
+Cluster : A named cluster of nodes for a service.
+
+
+== Design
+
+Each service’s backups are maintained independently. They may or may not run on independent or shared hardware. Nodes are numbered and linked to their aws instance id.
+A backup operation will validate the cluster name, environment and node name before proceeding with any mutation.
+
+=== Cluster Layout
+
+cluster.json
+  {
+    "environment": "production",
+    "name": "place_directory",
+    "nodes": {
+      "1": {"instance_id": "i-abcdefed" },
+      "2": {"instance_id": "i-abcdefec" },
+      "3": {"instance_id": "i-abcdefeb" }
+    }
+  }
+
+
+=== Data Layout
+
+General
+  /backup_bucket/name/environment/service/
+  /backup_bucket/name/environment/service/cluster.json
+  /backup_bucket/name/environment/service/full/datetime/node/
+  /backup_bucket/name/environment/service/full/datetime/node/_COMPLETED
+
+Cassandra
+  /cassandra/data/node/keyspace/table/place_directory_production-Places-ic-218008.tgz ...
+  /cassandra/full/datetime/node/contents.json
+  /cassandra/full/datetime/node/system.tar.gz
+  /cassandra/full/datetime/node/schema.cql
+  /cassandra/full/datetime/node/ring.txt
+
+ElasticSearch
+  /elasticsearch/full/datetime/node/cluster_state.local.json
+  /elasticsearch/full/datetime/node/cluster_state.global.json
+  /elasticsearch/full/datetime/node/index_name.restore.sh
+  /elasticsearch/full/datetime/node/index_name.tar.gz
+
+===Braavos Config
+
+cassandra.yml
+  name: place_directory
+  environment: production
+  service: cassandra
+  bucket_name: backup_bucket
+  data_loc: /var/lib/cassandra/data
+  sync_loc: /mnt/cassandra_snapshots/latest
+
+Operations
+
+Initiate a full backup: (IMPLEMENTED)
+  braavos -c cassandra.yml backup full
+
+List available backups: (IMPLEMENTED)
+  braavos -c cassandra.yml show
+Verify backup:
+  braavos -c cassandra.yml verify backup_loc
+
+Restore backup: (IMPLEMENTED)
+  braavos -c cassandra.yml restore backup_loc restore_loc
+
+Restore across clusters: (IMPLEMENTED)
+  braavos -c cassandra.yml restore --cluster 'name:environment:node_id' backup_loc restore_loc
+  Example: ./bin/braavos -c config/braavos.yml restore --cluster 'place_directory:production:1' full/20140321 /tmp/restore
+
+Prune backups:
+  braavos -c cassandra.yml prune
+
+Sync latest (copy to ebs):
+  braavos sync -c cassandra.yml
+
+== Mentions
+
+braavos uses and depends on the following awesome components:
+
+* {GNU Parallel}[http://www.gnu.org/software/parallel/] - The Command-Line Power Tool: O. Tange (2011)
+* {GNU coreutils}[http://www.gnu.org/software/coreutils/] - The GNU Core Utils, including timeout.
+* {s3cmd}[http://s3tools.org/s3cmd] - Command Line S3 Client. Please use version 1.5.0-beta1 or greater.
+* {Ruby}[https://www.ruby-lang.org/] - A Programmer's Best Friend (including ERB templating)

data/Rakefile

@@ -0,0 +1,23 @@
+require 'rake'
+require 'rake/testtask'
+
+require "bundler/gem_tasks"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = true
+end
+
+require 'rdoc/task'
+desc 'Generate documentation.'
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'Braavos Backup/Restore Administrator'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+desc 'Default: run unit tests.'
+task :default => :test

data/bin/braavos

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+# resolve bin path, ignoring symlinks
+require 'pathname'
+bin_file = Pathname.new(__FILE__).realpath
+
+# add locally vendored gems to libpath
+gem_dir = File.expand_path('../../vendor/gems', bin_file)
+Dir["#{gem_dir}/**/lib"].each do |libdir|
+  $:.unshift libdir
+end
+
+# add self to libpath
+$:.unshift File.expand_path('../../lib', bin_file)
+
+# start up the CLI
+require 'braavos/cli'
+exit Braavos::CLI.start(ARGV)
+

data/lib/braavos.rb

@@ -0,0 +1,179 @@
+ENV["RACK_ENV"] ||= "development"
+require 'bundler'
+
+require 'rake'
+require 'gli'
+require 'aws-sdk'
+
+require 'braavos/version'
+require 'yaml'
+require 'json'
+require 'logger'
+require 'set'
+
+class Braavos
+
+  autoload :Command, 'braavos/command'
+  autoload :Config, 'braavos/config'
+  autoload :Parallel, 'braavos/parallel'
+  autoload :Service, 'braavos/service'
+  autoload :Storage, 'braavos/storage'
+  autoload :Template, 'braavos/template'
+
+  class << self
+    def config(config={})
+      @config ||= Config.new(config)
+    end
+
+    def command
+      @command ||= Command.new
+    end
+
+    def parallel
+      @parallel ||= Parallel.new("parallel", config.parallel_jobs || "100%")
+    end
+
+    def template
+      @template ||= Template.new([File.expand_path("../../template", __FILE__)])
+    end
+
+    def service
+      @service ||= config.service_class.new
+    end
+
+    def storage
+      @storage ||= config.storage_class.new
+    end
+
+    def logger
+      @logger ||= Logger.new(STDOUT)
+    end
+  end
+
+  attr_reader :global_options
+
+  def initialize(options)
+    # Load config
+    Braavos.logger.info "Braavos v#{Braavos::VERSION} Initializing"
+    Braavos.config(YAML.load(ERB.new(File.read(options[:config])).result))
+
+    initialize_aws(options)
+
+    Braavos.logger.level = Logger::INFO
+    if options[:verbose]
+      Braavos.logger.level = Logger::DEBUG
+    end
+    @global_options = options
+    Braavos.logger.debug "Options: #{options}"
+    Braavos.logger.debug "Config: #{Braavos.config.settings}"
+    AWS.config(:logger => Braavos.logger, :log_level => :debug)
+  end
+
+  def backup_full(args, options={})
+    Braavos.storage.verify_cluster(true)
+    if Braavos.service.respond_to? :backup_full
+      Braavos.service.backup_full
+    else
+      raise "Operation not supported by service #{Braavos.config.service}"
+    end
+  end
+
+  def restore(args, options={})
+    Braavos.storage.verify_cluster(true)
+
+    # restore location must be empty
+    # backup location must contain _COMPLETED
+
+    if args.length != 2
+      raise "Invalid restore args: backup_loc restore_loc"
+    end
+
+    restore_loc = args[1]
+    if File.exists?(restore_loc) && !File.directory?(restore_loc)
+      raise "restore_loc #{restore_loc} is not a directory"
+    end
+
+    FileUtils.mkdir_p restore_loc
+
+    if Dir["#{restore_loc}/*"].size > 1
+      raise "restore_loc #{restore_loc} directory must be empty"
+    end
+
+    if options[:cluster]
+      if match = options[:cluster].match(/(\w+):(\w+):(\d+)/)
+        name, environment, node = match.captures
+        Braavos.config.name = name
+        Braavos.config.environment = environment
+        Braavos.config.backup_path(true)
+        Braavos.storage.find_node_id(node)
+      else
+        raise "Invalid cluster config"
+      end
+      Braavos.logger.info("Restore across cluster: #{options[:cluster]}")
+    end
+
+    Braavos.service.restore(args[0], args[1])
+
+  end
+
+  def show(args, options={})
+    Braavos.logger.info "Data ks: #{Braavos.service.keyspaces_data}"
+    Braavos.logger.info "System ks: #{Braavos.service.keyspaces_system}"
+
+    Braavos.storage.verify_cluster(false)
+    node_id = ( options[:node_id] || Braavos.storage.find_node_id )
+    Braavos.logger.info "Node id: #{node_id}"
+    listing = Braavos.storage.list_backups(node_id)
+    puts "AVAILABLE BACKUPS:"
+    listing.each do |g, l|
+      puts
+      puts g
+      puts '-' * g.size
+      if l.size == 0
+        puts "NONE"
+      else
+        l.each do |name|
+          puts "\t#{name}"
+        end
+      end
+    end
+  end
+
+  def verify(args, options={})
+  end
+
+  def prune(args, options={})
+
+  end
+
+  def sync(args, options={})
+    Braavos.service.sync
+  end
+
+  def cleanup
+
+  end
+
+  def initialize_aws(options)
+    default_path = File.expand_path("../../config/aws.yml", __FILE__)
+
+    if options[:aws]
+      initialize_aws_from_file(options[:aws])
+    elsif File.exists?(default_path)
+      initialize_aws_from_file(default_path)
+    else
+      initialize_aws_from_environment
+    end
+  end
+
+  def initialize_aws_from_environment
+    Braavos.config.settings['aws'] = AWS.config.credentials
+  end
+
+  def initialize_aws_from_file(path)
+    raise ArgumentError, "AWS Credentials file not found" unless File.exists?(path)
+    aws_config = YAML.load(ERB.new(File.read(path)).result)
+    AWS.config(aws_config)
+    Braavos.config.settings['aws'] = aws_config
+  end
+end

data/lib/braavos/cli.rb

@@ -0,0 +1,82 @@
+require 'gli'
+require 'braavos'
+
+class Braavos::CLI
+  extend GLI::App
+
+  class << self
+
+    def start(argv)
+      program_desc 'Backup/Restore Administrator'
+      sort_help :manually
+
+      version Braavos::VERSION
+
+      flag [:c,:config], desc: 'Configuration file', arg_name: 'config', required: true
+      flag [:a,:aws], desc: 'AWS Credentials file', arg_name: 'aws', required: false
+      switch [:v,:verbose], desc: 'Verbose mode'
+
+      pre do |global,command,options,args|
+        @braavos = Braavos.new(global)
+        true
+      end
+
+      desc 'Execute Backup'
+      command :backup do |c|
+        c.action do |global_options,options,args|
+          @braavos.backup_full(args, options)
+        end
+      end
+
+      desc 'Execute Restore'
+      command :restore do |c|
+        c.flag [:cluster], desc: 'Restore across cluster. Format: "cluster:environment:node"'
+
+        c.arg_name 'timestamp'
+        c.action do |global_options,options,args|
+          @braavos.restore(args, options)
+        end
+      end
+
+      desc 'Show Operation'
+      command :show do |c|
+        c.action do |global_options,options,args|
+          @braavos.show(args, options)
+        end
+      end
+
+      desc 'Verify Operation'
+      command :verify do |c|
+        c.action do |global_options,options,args|
+          @braavos.verify(args, options)
+        end
+      end
+
+      desc 'Sync Operation'
+      command :sync do |c|
+        c.action do |global_options,options,args|
+          @braavos.sync(args, options)
+        end
+      end
+
+      desc 'Prune Operation'
+      command :prune do |c|
+        c.action do |global_options,options,args|
+          @braavos.prune(args, options)
+        end
+      end
+
+      post do |global,command,options,args|
+        @braavos.cleanup
+      end
+
+      on_error do |exception|
+        @braavos.cleanup if @braavos
+        raise exception unless exception.is_a?(GLI::BadCommandLine)
+        true
+      end
+
+      run(argv)
+    end
+  end
+end