braavos 0.0.7

data/lib/braavos/storage/file.rb

@@ -0,0 +1,48 @@
+class Braavos::Storage::File < Braavos::Storage::StorageBase
+
+  def load_file(location)
+    Braavos.logger.debug("load_file: #{location}")
+    File.read(location)
+  end
+
+  # Returns a recursive listing of {"name" => true} if directory and {"name" => *file size*} if file
+  def list_dir(location)
+    Braavos.logger.debug("list_dir: #{location}")
+    listing = Dir["#{location}/**/*"].inject({}) do |h,i|
+      h[i] = File.size?(i) if not File.directory?(i)
+      h
+    end
+    Braavos.logger.debug("\t#{listing}")
+    listing
+  end
+
+  def clear_result(location)
+    FileUtils.rm_f File.join(location, "_FAILED")
+  end
+
+  def has_success?(location)
+    path = File.join(location, "_COMPLETED")
+    list_dir(location).size > 0 &&
+      File.exists?(path)
+  end
+
+  def script_path(location)
+    @mkdir_cache ||= Hash.new do |h, key|
+      FileUtils.mkdir_p(key)
+      h[key] = true
+    end
+
+    @mkdir_cache[File.dirname(location)]
+    location
+  end
+
+  def write_file(location, contents)
+    script_path(location) # for mkdir -p
+    if contents.is_a?(Hash) && contents.include?(:file)
+      FileUtils.cp(contents[:file], location)
+    else
+      File.open(location, 'w') do |f| f.write contents end
+    end
+  end
+
+end

data/lib/braavos/storage/s3.rb

@@ -0,0 +1,41 @@
+class Braavos::Storage::S3 < Braavos::Storage::StorageBase
+
+  attr_reader :s3
+
+  def initialize
+    @s3 = AWS::S3.new
+  end
+
+  def load_file(location)
+    s3.buckets[Braavos.config.bucket_name].objects[location].read
+  end
+
+  def list_dir(location)
+    Braavos.logger.debug("S3 list_dir: #{Braavos.config.bucket_name}/#{location}")
+    listing = s3.buckets[Braavos.config.bucket_name].objects.with_prefix("#{location}/").inject({}) do |h,i|
+      h[i.key] = 0 if not i.key.end_with?('_$folder$') # disable i.content_length due to performance
+      h
+    end
+    Braavos.logger.debug("\t#{listing}")
+    listing
+  end
+
+  def has_success?(location)
+    path = File.join(location, "_COMPLETED")
+    list_dir(location).size > 0 &&
+      s3.buckets[Braavos.config.bucket_name].objects[path].exists?
+  end
+
+  def clear_result(location)
+    path = File.join(location, "_FAILED")
+    s3.buckets[Braavos.config.bucket_name].objects[path].delete
+  end
+
+  def script_path(location)
+    File.join(Braavos.config.bucket_name, location)
+  end
+
+  def write_file(location, contents)
+    s3.buckets[Braavos.config.bucket_name].objects[location].write(contents)
+  end
+end

data/lib/braavos/storage/storage_base.rb

@@ -0,0 +1,48 @@
+class Braavos::Storage::StorageBase
+
+
+  def get_cluster
+    @cluster ||= JSON.parse(load_file(File.join(Braavos.config.backup_path, 'cluster.json')))
+  end
+
+  def verify_cluster(write_mode)
+    cluster = get_cluster
+    if write_mode
+      if Braavos.config.name != cluster['name']
+        raise "Mismatching name (expecting: #{Braavos.config.name}, found: #{cluster['name']})"
+      end
+      if Braavos.config.environment != cluster['environment']
+        raise "Mismatching environment (expecting: #{Braavos.config.environment}, found: #{cluster['environment']})"
+      end
+    end
+  end
+
+  def find_node_id(force_node_id=nil)
+    @node_id = force_node_id if force_node_id
+    @node_id ||= -> do
+      cluster = get_cluster
+      instance_id = Braavos::Service.instance_id
+      cluster['nodes'].each do |node_id, node_info|
+        if node_info['instance_id'] == instance_id
+          return node_id
+        end
+      end
+      raise "Node not found: #{instance_id}"
+    end.call
+  end
+
+  # TODO: option for --completed --pending, maybe return size here
+  def list_backups(node_id)
+    result = {full: [], incr: []}
+    listing = list_dir(Braavos.config.backup_path)
+    listing.each do |l, z|
+      if match = l.match(/\/(full|incr)\/(\w+)\/#{node_id}\/_COMPLETED\Z/)
+        path, name = match.captures
+        result[path.to_sym] << name
+      end
+    end
+    result
+  end
+
+
+end

data/lib/braavos/template.rb

@@ -0,0 +1,49 @@
+require 'erb'
+
+class Braavos::Template
+
+  attr_accessor :template_paths
+
+  def initialize(template_paths)
+    @template_paths = template_paths
+  end
+
+  def load_template(template, params={})
+    content = find_template(template)
+    raise "Unable to find template: #{template} in path: #{template_paths}" unless content
+    result_template(content, params)
+  end
+
+  def result_template(template, params)
+    ns = Namespace.new(params.merge(config: Braavos.config))
+    ERB.new(template).result(ns.get_binding)
+  end
+
+  private
+
+  # http://stackoverflow.com/questions/1338960/ruby-templates-how-to-pass-variables-into-inlined-erb
+  class Namespace
+    def initialize(hash)
+      hash.each do |key, value|
+        singleton_class.send(:define_method, key) { value }
+      end
+    end
+
+    def get_binding
+      binding
+    end
+  end
+
+  def find_template(template)
+    content = nil
+    template_paths.each do |path|
+      file_path = File.join(path, template)
+      if File.exists?(file_path)
+        content = File.read(file_path)
+        break
+      end
+    end
+    content
+  end
+
+end

data/lib/braavos/version.rb

@@ -0,0 +1,3 @@
+class Braavos
+  VERSION = "0.0.7"
+end

data/template/cassandra/clear_snapshot.sh.erb

@@ -0,0 +1 @@
+exec nodetool clearsnapshot $1 -t $2

data/template/cassandra/create_snapshot.sh.erb

@@ -0,0 +1 @@
+exec nodetool snapshot $1 -t $2

data/template/cassandra/dump_schema.sh.erb

@@ -0,0 +1,4 @@
+KEYSPACE=$1
+OUTPUT_LOC=$2
+
+echo "describe keyspace $KEYSPACE" | cqlsh <%= local_host_ip %> > $OUTPUT_LOC/$KEYSPACE.cql

data/template/cassandra/system_bundle.sh.erb

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -e
+
+DEST_DIR=$1
+shift
+DATA_DIR=$1
+shift
+
+cd ${DATA_DIR} && tar zcf ${DEST_DIR}/system.tgz $*
+
+nodetool ring > ${DEST_DIR}/ring.txt
+nodetool info > ${DEST_DIR}/info.txt
+nodetool cfstats > ${DEST_DIR}/cfstats.txt
+nodetool status > ${DEST_DIR}/status.txt
+nodetool compactionstats > ${DEST_DIR}/compactionstats.txt
+nodetool netstats > ${DEST_DIR}/netstats.txt
+nodetool gossipinfo > ${DEST_DIR}/gossipinfo.txt

data/template/cassandra/table_bundle_restore.sh.erb

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+set -e
+
+FILE_LOC=$1
+DEST_LOC=$2
+PROCESS_LOC=${FILE_LOC}
+
+<% if config.storage_backing == 's3' %>
+
+PROCESS_LOC=<%= config.temp_loc %>/braavos-cassdown-$$.${RANDOM}.tgz
+
+function getTimeoutCmd() {
+  if hash timeout &> /dev/null
+  then
+    echo timeout
+  else
+    echo gtimeout
+  fi
+}
+
+trap "echo Caught kill for ${FILE_LOC}; rm ${PROCESS_LOC}; exit 1" SIGHUP SIGINT SIGTERM
+
+# Install GNU coreutils for `timeout`
+# s3cmd can hang indefinitely due to network constraints
+# So kill after timeout of 2 minutes, an ideal timeout when run on EC2
+$(getTimeoutCmd) 120 s3cmd get s3://${FILE_LOC} ${PROCESS_LOC}
+if [ $? -eq 124 ]; then
+    # Timeout occurred, try one more time.
+    echo Timed Out! Retrying download of ${FILE_LOC}
+    s3cmd get s3://${FILE_LOC} ${PROCESS_LOC}
+fi
+
+<% end %>
+
+echo Expanding ${FILE_LOC} to ${DEST_LOC}
+mkdir -p ${DEST_LOC}
+zcat -q ${PROCESS_LOC} | tar -xC ${DEST_LOC}
+
+<% if config.storage_backing == 's3' %>
+
+rm ${PROCESS_LOC}
+
+<% end %>

data/template/cassandra/table_bundle_upload.sh.erb

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+set -e
+
+FILE_LOC=$1
+START_LOC=$(pwd)
+DEST_LOC=$2
+TMP_FILE=<%= config.temp_loc %>/braavos-cassup-$$.${RANDOM}.tgz
+
+trap "echo Caught kill for ${FILE_LOC}; rm ${TMP_FILE}; exit 1" SIGHUP SIGINT SIGTERM
+
+BASE=$(dirname $FILE_LOC)
+FILE_PART=$(basename $FILE_LOC)
+
+echo Processing $FILE_PART in $BASE to ${DEST_LOC}
+cd $BASE && tar zcf $TMP_FILE ${FILE_PART}-*
+
+<% if config.storage_backing == 'file' %>
+
+mv $TMP_FILE $START_LOC/$DEST_LOC
+
+<% elsif config.storage_backing == 's3' %>
+
+s3cmd put ${TMP_FILE} s3://${DEST_LOC}
+rm ${TMP_FILE}
+
+<% end %>

data/test/braavos/command_test.rb

@@ -0,0 +1,34 @@
+require 'test_helper'
+
+class Braavos::CommandTest < MiniTest::Spec
+  attr_accessor :command, :file
+
+  before do
+    Braavos.logger.level = Logger::WARN
+    @file = Tempfile.new "braavos_command_test"
+    @command = Braavos::Command.new
+  end
+
+  after do
+    file.close
+    file.unlink
+  end
+
+  def test_simple
+    command.execute("echo $* >> #{file.path}", ['a', 'b', 'c'])
+    result = file.read
+    assert_equal "a b c\n", result
+  end
+
+  def test_arguments
+    command.execute("echo $1 >> #{file.path}", ['a', 'b', 'c'])
+    result = file.read
+    assert_equal "a\n", result
+  end
+
+  def test_system
+    command.execute("echo $* >> #{file.path}", ['a', 'b', 'c'])
+    result = file.read
+    assert result
+  end
+end

data/test/braavos/config_test.rb

@@ -0,0 +1,66 @@
+require 'test_helper'
+
+class Braavos::ConfigTest < MiniTest::Spec
+
+  def default_config
+    {temp_loc: '/tmp'}
+  end
+
+  def test_regular_backup_path
+    c = default_config.merge(name: 'neville', environment: 'env', service: 'cassandra')
+    config = Braavos::Config.new(c)
+
+    assert_equal "neville/env/cassandra", config.backup_path
+  end
+
+  def test_prefixed_backup_path
+    c = default_config.merge(name: 'neville', environment: 'env', service: 'cassandra', backup_prefix: '/Users/nevillek/backup/')
+    config = Braavos::Config.new(c)
+
+    assert_equal "/Users/nevillek/backup/neville/env/cassandra", config.backup_path
+
+    c = default_config.merge(name: 'neville', environment: 'env', service: 'cassandra', backup_prefix: 'Users/nevillek/backup')
+    config = Braavos::Config.new(c)
+
+    assert_equal "Users/nevillek/backup/neville/env/cassandra", config.backup_path
+
+    c = default_config.merge(name: 'neville', environment: 'env', service: 'cassandra', backup_prefix: '')
+    config = Braavos::Config.new(c)
+
+    assert_equal "neville/env/cassandra", config.backup_path
+  end
+
+  def test_service_cassandra
+    c = default_config.merge(name: 'neville', environment: 'env', service: 'cassandra')
+    config = Braavos::Config.new(c)
+
+    assert config.service_class == Braavos::Service::Cassandra
+  end
+
+  def test_service_es
+    c = default_config.merge(name: 'neville', environment: 'env', service: 'elasticsearch')
+    config = Braavos::Config.new(c)
+
+    assert config.service_class == Braavos::Service::Elasticsearch
+  end
+
+  def test_storage_s3
+    c = default_config.merge(name: 'neville', environment: 'env', service: 'cassandra')
+    config = Braavos::Config.new(c)
+
+    assert config.storage_class == Braavos::Storage::S3
+
+    c = default_config.merge(name: 'neville', environment: 'env', service: 'cassandra', storage_backing: 's3')
+    config = Braavos::Config.new(c)
+
+    assert config.storage_class == Braavos::Storage::S3
+  end
+
+  def test_storage_file
+    c = default_config.merge(name: 'neville', environment: 'env', service: 'cassandra', storage_backing: 'file')
+    config = Braavos::Config.new(c)
+
+    assert config.storage_class == Braavos::Storage::File
+  end
+
+end

data/test/braavos/parallel_test.rb

@@ -0,0 +1,29 @@
+require 'test_helper'
+
+class Braavos::ParallelTest < MiniTest::Spec
+  attr_accessor :parallel, :file
+
+  before do
+    Braavos.logger.level = Logger::WARN
+    @file = Tempfile.new "braavos_parallel_test"
+    @parallel = Braavos::Parallel.new
+  end
+
+  after do
+    file.close
+    file.unlink
+  end
+
+  def test_simple
+    result = parallel.execute("echo $1 >> #{file.path}", ['a', 'b', 'c'])
+    result = file.read
+    assert_equal ['a', 'b', 'c'], result.lines.map(&:chomp).sort
+  end
+
+  def test_multiple_inputs
+    parallel.execute("echo $* >> #{file.path}", [['a', 'apple'], ['b', 'ball'], ['c', 'crawl']])
+    result = file.read
+    assert_equal ['a apple', 'b ball', 'c crawl'], result.lines.map(&:chomp).sort
+  end
+
+end

data/test/braavos/template_test.rb

@@ -0,0 +1,36 @@
+require 'test_helper'
+
+class Braavos::TemplateTest < MiniTest::Spec
+  attr_accessor :template
+
+  before do
+    @template = Braavos::Template.new(['test/template'])
+  end
+
+  def test_result_simple
+    result = template.result_template('Hello <%= name %>, Welcome to <%= place %>!', name: 'Neville', place: 'Home')
+
+    assert_equal 'Hello Neville, Welcome to Home!', result
+  end
+
+  def test_result_missing
+
+    assert_raises NameError do
+      result = template.result_template('Hello <%= name %>, Welcome to <%= place %>!', place: 'Home')
+    end
+
+  end
+
+  def test_simple_template
+    result = template.load_template('simple.txt.erb', name: 'Neville', place: 'Home')
+
+    assert_equal 'Hello Neville, Welcome to Home!', result
+  end
+
+  def test_nested_dir
+    result = template.load_template('nested/inner.sh.erb')
+
+    assert_equal "#!/bin/bash", result
+  end
+
+end