netguru-safe 0.2.9

data/VERSION

@@ -0,0 +1 @@
+0.2.9.pre

data/astrails-safe.gemspec

@@ -0,0 +1,37 @@
+require './lib/astrails/safe/version'
+
+Gem::Specification.new do |s|
+  s.name        = %q{netguru-safe}
+  s.version     = Astrails::Safe::VERSION
+  s.authors     = ["Astrails Ltd."]
+  s.date        = Time.now.utc.strftime("%Y-%m-%d")
+  s.email       = %q{we@astrails.com}
+  s.homepage    = %q{http://blog.astrails.com/astrails-safe}
+  s.summary     = %q{Backup filesystem and databases (MySQL and PostgreSQL) locally or to a remote server/service (with encryption)}
+  s.description = %q{Astrails-Safe is a simple tool to backup databases (MySQL and PostgreSQL), Subversion repositories (with svndump) and just files.
+Backups can be stored locally or remotely and can be enctypted.
+Remote storage is supported on Amazon S3, Rackspace Cloud Files, or just plain SFTP.
+}
+
+  s.extra_rdoc_files = [
+    "LICENSE",
+    "README.markdown",
+    "TODO"
+  ]
+
+  s.files                     = `git ls-files`.split("\n")
+  s.test_files                = `git ls-files spec`.split("\n")
+  s.require_paths             = ["lib"]
+  s.required_rubygems_version = %q{1.5.0}
+  s.default_executable        = %q{astrails-safe}
+  s.executables               = ["astrails-safe"]
+
+  # tests
+  s.add_development_dependency 'rspec', '~> 1.3.2'
+  s.add_development_dependency 'rr', '~> 1.0.4'
+
+  s.add_runtime_dependency 'aws-sdk', '~> 1.2.3'
+  s.add_runtime_dependency 'cloudfiles', '~> 1.4.7'
+  s.add_runtime_dependency 'net-sftp', '~> 2.0.4'
+  s.add_runtime_dependency 'toadhopper', '~> 2.0'
+end

data/bin/astrails-safe

@@ -0,0 +1,53 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+
+# require 'ruby-debug'
+$:.unshift File.expand_path("../../lib", __FILE__)
+
+require 'astrails/safe'
+include Astrails::Safe
+
+def die(msg)
+  puts "ERROR: #{msg}"
+  exit 1
+end
+
+def usage
+  puts <<-END
+Usage: astrails-safe [OPTIONS] CONFIG_FILE
+Options:
+  -h, --help           This help screen
+  -v, --verbose        be verbose, duh!
+  -n, --dry-run        just pretend, don't do anything.
+  -L, --local          skip S3 and Cloud Files
+
+Note: config file will be created from template if missing
+END
+  exit 1
+end
+
+def process_options
+  usage if ARGV.delete("-h") || ARGV.delete("--help")
+  $_VERBOSE = ARGV.delete("-v") || ARGV.delete("--verbose")
+  $DRY_RUN = ARGV.delete("-n") || ARGV.delete("--dry-run")
+  $LOCAL   = ARGV.delete("-L") || ARGV.delete("--local")
+  usage unless ARGV.first
+  $CONFIG_FILE_NAME = File.expand_path(ARGV.first)
+end
+
+def main
+  process_options
+
+  unless File.exists?($CONFIG_FILE_NAME)
+    die "Missing configuration file. NOT CREATED! Rerun w/o the -n argument to create a template configuration file." if $DRY_RUN
+
+    FileUtils.cp File.join(Astrails::Safe::ROOT, "templates", "script.rb"), $CONFIG_FILE_NAME
+
+    die "Created default #{$CONFIG_FILE_NAME}. Please edit and run again."
+  end
+
+  load($CONFIG_FILE_NAME)
+end
+
+main

data/examples/example_helper.rb

@@ -0,0 +1,19 @@
+require 'rubygems'
+require 'micronaut'
+require 'ruby-debug'
+
+SAFE_ROOT = File.dirname(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(SAFE_ROOT, 'lib'))
+
+require 'astrails/safe'
+
+def not_in_editor?
+  !(ENV.has_key?('TM_MODE') || ENV.has_key?('EMACS') || ENV.has_key?('VIM'))
+end
+
+Micronaut.configure do |c|
+  c.color_enabled = not_in_editor?
+  c.filter_run :focused => true
+  c.mock_with :rr
+end

data/lib/astrails/safe.rb

@@ -0,0 +1,73 @@
+require "aws-sdk"
+require "cloudfiles"
+require 'net/sftp'
+require 'fileutils'
+require 'benchmark'
+require 'toadhopper'
+
+require 'tempfile'
+require 'extensions/mktmpdir'
+
+require 'astrails/safe/tmp_file'
+
+require 'astrails/safe/config/node'
+require 'astrails/safe/config/builder'
+
+require 'astrails/safe/stream'
+
+require 'astrails/safe/backup'
+
+require 'astrails/safe/source'
+require 'astrails/safe/mysqldump'
+require 'astrails/safe/pgdump'
+require 'astrails/safe/archive'
+require 'astrails/safe/svndump'
+require 'astrails/safe/mongodump'
+
+require 'astrails/safe/pipe'
+require 'astrails/safe/gpg'
+require 'astrails/safe/gzip'
+
+require 'astrails/safe/sink'
+require 'astrails/safe/local'
+require 'astrails/safe/s3'
+require 'astrails/safe/cloudfiles'
+require 'astrails/safe/sftp'
+
+require 'astrails/safe/version'
+
+module Astrails
+  module Safe
+    ROOT = File.join(File.dirname(__FILE__), "..", "..")
+
+    def safe(&block)
+      config = Config::Node.new(&block)
+
+      begin
+        [[Mysqldump, [:mysqldump, :databases]],
+         [Pgdump,    [:pgdump,    :databases]],
+         [Mongodump, [:mongodump, :databases]],
+         [Archive,   [:tar,       :archives]],
+         [Svndump,   [:svndump,   :repos]]
+        ].each do |klass, path|
+          if collection = config[*path]
+            collection.each do |name, config|
+              klass.new(name, config).backup.run(config, :gpg, :gzip, :local, :s3, :cloudfiles, :sftp)
+            end
+          end
+        end
+      rescue => e
+        begin
+          if config["airbrake"]
+            toad = Toadhopper.new(config["airbrake"]["api_key"])
+            toad.post!(e)
+          end
+        rescue
+        end
+      ensure
+        Astrails::Safe::TmpFile.cleanup
+      end
+    end
+    module_function :safe
+  end
+end

data/lib/astrails/safe/archive.rb

@@ -0,0 +1,24 @@
+module Astrails
+  module Safe
+    class Archive < Source
+
+      def command
+        "tar -cf - #{@config[:options]} #{tar_exclude_files} #{tar_files}"
+      end
+
+      def extension; '.tar'; end
+
+      protected
+
+      def tar_exclude_files
+        [*@config[:exclude]].compact.map{|x| "--exclude=#{x}"}.join(" ")
+      end
+
+      def tar_files
+        raise RuntimeError, "missing files for tar" unless @config[:files]
+        [*@config[:files]].map{|s| s.strip}.join(" ")
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/backup.rb

@@ -0,0 +1,20 @@
+module Astrails
+  module Safe
+    class Backup
+      attr_accessor :id, :kind, :filename, :extension, :command, :compressed, :timestamp, :path
+      def initialize(opts = {})
+        opts.each do |k, v|
+          self.send("#{k}=", v)
+        end
+      end
+
+      def run(config, *mods)
+        mods.each do |mod|
+          mod = mod.to_s
+          mod[0] = mod[0..0].upcase
+          Astrails::Safe.const_get(mod).new(config, self).process
+        end
+      end
+    end
+  end
+end

data/lib/astrails/safe/cloudfiles.rb

@@ -0,0 +1,77 @@
+module Astrails
+  module Safe
+    class Cloudfiles < Sink
+      MAX_CLOUDFILES_FILE_SIZE = 5368709120
+
+      protected
+
+      def active?
+        container && user && api_key
+      end
+
+      def path
+        @path ||= expand(config[:cloudfiles, :path] || config[:local, :path] || ":kind/:id")
+      end
+
+      # UGLY: we need this function for the reason that
+      # we can't double mock on ruby 1.9.2, duh!
+      # so we created this func to mock it all together
+      def get_file_size(path)
+        File.stat(path).size
+      end
+
+      def save
+        raise RuntimeError, "pipe-streaming not supported for S3." unless @backup.path
+
+        # needed in cleanup even on dry run
+        cf = CloudFiles::Connection.new(user, api_key, true, service_net) unless $LOCAL
+        puts "Uploading #{container}:#{full_path} from #{@backup.path}" if $_VERBOSE || $DRY_RUN
+        unless $DRY_RUN || $LOCAL
+          if get_file_size(@backup.path) > MAX_CLOUDFILES_FILE_SIZE
+            STDERR.puts "ERROR: File size exceeds maximum allowed for upload to Cloud Files (#{MAX_CLOUDFILES_FILE_SIZE}): #{@backup.path}"
+            return
+          end
+          benchmark = Benchmark.realtime do
+            cf_container = cf.create_container(container)
+            o = cf_container.create_object(full_path,true)
+            o.write(File.open(@backup.path))
+          end
+          puts "...done" if $_VERBOSE
+          puts("Upload took " + sprintf("%.2f", benchmark) + " second(s).") if $_VERBOSE
+        end
+      end
+
+      def cleanup
+        return if $LOCAL
+
+        return unless keep = @config[:keep, :cloudfiles]
+
+        puts "listing files: #{container}:#{base}*" if $_VERBOSE
+        cf = CloudFiles::Connection.new(user, api_key, true, service_net) unless $LOCAL
+        cf_container = cf.container(container)
+        files = cf_container.objects(:prefix => base).sort
+
+        cleanup_with_limit(files, keep) do |f|
+          puts "removing Cloud File #{container}:#{f}" if $DRY_RUN || $_VERBOSE
+          cf_container.delete_object(f) unless $DRY_RUN || $LOCAL
+        end
+      end
+
+      def container
+        @config[:cloudfiles, :container]
+      end
+
+      def user
+        @config[:cloudfiles, :user]
+      end
+
+      def api_key
+        @config[:cloudfiles, :api_key]
+      end
+
+      def service_net
+        @config[:cloudfiles, :service_net] || false
+      end
+    end
+  end
+end

data/lib/astrails/safe/config/builder.rb

@@ -0,0 +1,60 @@
+module Astrails
+  module Safe
+    module Config
+      class Builder
+        COLLECTIONS = %w/database archive repo/
+        ITEMS = %w/s3 cloudfiles key secret bucket api_key container service_net path gpg password keep local mysqldump pgdump command options
+        user host port socket skip_tables tar files exclude filename svndump repo_path sftp mongodump airbrake/
+        NAMES = COLLECTIONS + ITEMS
+        def initialize(node)
+          @node = node
+        end
+
+        # supported args:
+        #   args = [value]
+        #   args = [id, data]
+        #   args = [data]
+        # id/value - simple values, data - hash
+        def method_missing(sym, *args, &block)
+          return super unless NAMES.include?(sym.to_s)
+
+          # do we have id or value?
+          unless args.first.is_a?(Hash)
+            id_or_value = args.shift # nil for args == []
+          end
+
+          id_or_value = id_or_value.map {|v| v.to_s} if id_or_value.is_a?(Array)
+
+          # do we have data hash?
+          if data = args.shift
+            raise "#{sym}: hash expected: #{data.inspect}" unless data.is_a?(Hash)
+          end
+
+          #puts "#{sym}: args=#{args.inspect}, id_or_value=#{id_or_value}, data=#{data.inspect}, block=#{block.inspect}"
+
+          raise "#{sym}: unexpected: #{args.inspect}" unless args.empty?
+          raise "#{sym}: missing arguments" unless id_or_value || data || block
+
+          if COLLECTIONS.include?(sym.to_s) && id_or_value
+            data ||= {}
+          end
+
+          if !data && !block
+            # simple value assignment
+            @node[sym] = id_or_value
+
+          elsif id_or_value
+            # collection element with id => create collection node and a subnode in it
+            key = sym.to_s + "s"
+            collection = @node[key] || @node.set(key, {})
+            collection.set(id_or_value, data || {}, &block)
+
+          else
+            # simple subnode
+            @node.set(sym, data || {}, &block)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/astrails/safe/config/node.rb

@@ -0,0 +1,76 @@
+require 'astrails/safe/config/builder'
+module Astrails
+  module Safe
+    module Config
+      class Node
+        attr_reader :parent
+        attr_reader :data
+        def initialize(parent = nil, data = {}, &block)
+          @parent, @data = parent, {}
+          data.each { |k, v| self[k] = v }
+          Builder.new(self).instance_eval(&block) if block
+        end
+
+        # looks for the path from this node DOWN. will not delegate to parent
+        def get(*path)
+          key = path.shift
+          value = @data[key.to_s]
+          return value if value && path.empty?
+
+          value && value.get(*path)
+        end
+
+        # recursive find
+        # starts at the node and continues to the parent
+        def find(*path)
+          get(*path) || @parent && @parent.find(*path)
+        end
+        alias :[] :find
+
+        MULTIVALUES = %w/skip_tables exclude files/
+        def set(key, value, &block)
+          if @data[key.to_s]
+            raise(ArgumentError, "duplicate value for '#{key}'") if value.is_a?(Hash) || !MULTIVALUES.include?(key.to_s)
+          end
+
+          if value.is_a?(Hash)
+            @data[key.to_s] = Node.new(self, value, &block)
+          else
+            raise(ArgumentError, "#{key}: no block supported for simple values") if block
+            if @data[key.to_s]
+              @data[key.to_s] = [*@data[key.to_s]] + [value]
+            else
+              @data[key.to_s] = value
+            end
+            value
+          end
+        end
+        alias :[]= :set
+
+        def each(&block)
+          @data.each(&block)
+        end
+        include Enumerable
+
+        def to_hash
+          @data.keys.inject({}) do |res, key|
+            value = @data[key]
+            res[key] = value.is_a?(Node) ? value.to_hash : value
+            res
+          end
+        end
+
+        def dump(indent = "")
+          @data.each do |key, value|
+            if value.is_a?(Node)
+              puts "#{indent}#{key}:"
+              value.dump(indent + "    ")
+            else
+              puts "#{indent}#{key}: #{value.inspect}"
+            end
+          end
+        end
+      end
+    end
+  end
+end