markmansour-safe 0.1.7

data/examples/unit/stream_example.rb

@@ -0,0 +1,33 @@
+require File.expand_path(File.dirname(__FILE__) + '/../example_helper')
+
+describe Astrails::Safe::Stream do
+
+  before(:each) do
+    @parent = Object.new
+    @stream = Astrails::Safe::Stream.new(@parent)
+    @r = rand(10)
+  end
+
+  def self.it_delegates_to_parent(prop)
+    it "delegates #{prop} to parent if not set" do
+      mock(@parent).__send__(prop) {@r}
+      @stream.send(prop).should == @r
+    end
+  end
+
+  def self.it_delegates_to_parent_with_cache(prop)
+    it_delegates_to_parent(prop)
+
+    it "uses cached value for #{prop}" do
+      dont_allow(@parent).__send__(prop)
+      @stream.instance_variable_set "@#{prop}", @r + 1
+      @stream.send(prop).should == @r + 1
+    end
+  end
+
+  it_delegates_to_parent_with_cache :id
+  it_delegates_to_parent_with_cache :config
+
+  it_delegates_to_parent :filename
+
+end

data/lib/astrails/safe.rb

@@ -0,0 +1,43 @@
+require 'extensions/mktmpdir'
+require 'astrails/safe/tmp_file'
+
+require 'astrails/safe/config/node'
+require 'astrails/safe/config/builder'
+
+require 'astrails/safe/stream'
+
+require 'astrails/safe/source'
+require 'astrails/safe/mysqldump'
+require 'astrails/safe/pgdump'
+require 'astrails/safe/archive'
+
+require 'astrails/safe/pipe'
+require 'astrails/safe/gpg'
+require 'astrails/safe/gzip'
+
+require 'astrails/safe/sink'
+require 'astrails/safe/local'
+require 'astrails/safe/s3'
+
+
+module Astrails
+  module Safe
+    ROOT = File.join(File.dirname(__FILE__), "..", "..")
+
+    def timestamp
+      @timestamp ||= Time.now.strftime("%y%m%d-%H%M")
+    end
+
+    def safe(&block)
+      config = Config::Node.new(&block)
+      #config.dump
+
+      Astrails::Safe::Mysqldump.run(config[:mysqldump, :databases])
+      Astrails::Safe::Pgdump.run(config[:pgdump, :databases])
+      Astrails::Safe::Archive.run(config[:tar, :archives])
+
+      Astrails::Safe::TmpFile.cleanup
+    end
+  end
+end
+

data/lib/astrails/safe/archive.rb

@@ -0,0 +1,24 @@
+module Astrails
+  module Safe
+    class Archive < Source
+
+      def command
+        "tar -cf - #{@config[:options]} #{tar_exclude_files} #{tar_files}"
+      end
+
+      def extension; '.tar'; end
+
+      protected
+
+      def tar_exclude_files
+        [*@config[:exclude]].compact.map{|x| "--exclude=#{x}"} * " "
+      end
+
+      def tar_files
+        raise RuntimeError, "missing files for tar" unless @config[:files]
+        [*@config[:files]] * " "
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/config/builder.rb

@@ -0,0 +1,60 @@
+module Astrails
+  module Safe
+    module Config
+      class Builder
+        COLLECTIONS = %w/database archive/
+        ITEMS = %w/s3 key secret bucket path gpg password keep local mysqldump pgdump options
+        user host port socket skip_tables tar files exclude filename/
+        NAMES = COLLECTIONS + ITEMS
+        def initialize(node)
+          @node = node
+        end
+
+        # supported args:
+        #   args = [value]
+        #   args = [id, data]
+        #   args = [data]
+        # id/value - simple values, data - hash
+        def method_missing(sym, *args, &block)
+          return super unless NAMES.include?(sym.to_s)
+
+          # do we have id or value?
+          unless args.first.is_a?(Hash)
+            id_or_value = args.shift # nil for args == []
+          end
+
+          id_or_value = id_or_value.map {|v| v.to_s} if id_or_value.is_a?(Array)
+
+          # do we have data hash?
+          if data = args.shift
+            die "#{sym}: hash expected: #{data.inspect}" unless data.is_a?(Hash)
+          end
+
+          #puts "#{sym}: args=#{args.inspect}, id_or_value=#{id_or_value}, data=#{data.inspect}, block=#{block.inspect}"
+
+          die "#{sym}: unexpected: #{args.inspect}" unless args.empty?
+          die "#{sym}: missing arguments" unless id_or_value || data || block
+
+          if COLLECTIONS.include?(sym.to_s) && id_or_value
+            data ||= {}
+          end
+
+          if !data && !block
+            # simple value assignment
+            @node[sym] = id_or_value
+
+          elsif id_or_value
+            # collection element with id => create collection node and a subnode in it
+            key = sym.to_s + "s"
+            collection = @node[key] || @node.set(key, {})
+            collection.set(id_or_value, data || {}, &block)
+
+          else
+            # simple subnode
+            @node.set(sym, data || {}, &block)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/astrails/safe/config/node.rb

@@ -0,0 +1,67 @@
+require 'astrails/safe/config/builder'
+module Astrails
+  module Safe
+    module Config
+      class Node
+        attr_reader :parent
+        def initialize(parent = nil, data = {}, &block)
+          @parent, @data = parent, {}
+          data.each { |k, v| self[k] = v }
+          Builder.new(self).instance_eval(&block) if block
+        end
+
+        # looks for the path from this node DOWN. will not delegate to parent
+        def get(*path)
+          key = path.shift
+          value = @data[key.to_s]
+          return value if value && path.empty?
+
+          value && value.get(*path)
+        end
+
+        # recursive find
+        # starts at the node and continues to the parent
+        def find(*path)
+          get(*path) || @parent && @parent.find(*path)
+        end
+        alias :[] :find
+
+        def set(key, value, &block)
+          @data[key.to_s] =
+            if value.is_a?(Hash)
+              Node.new(self, value, &block)
+            else
+              raise(ArgumentError, "#{key}: no block supported for simple values") if block
+              value
+            end
+        end
+        alias :[]= :set
+
+        def each(&block)
+          @data.each(&block)
+        end
+        include Enumerable
+
+        def to_hash
+          @data.keys.inject({}) do |res, key|
+            value = @data[key]
+            res[key] = value.is_a?(Node) ? value.to_hash : value
+            res
+          end
+        end
+
+        def dump(indent = "")
+          @data.each do |key, value|
+            if value.is_a?(Node)
+              puts "#{indent}#{key}:"
+              value.dump(indent + "    ")
+            else
+              puts "#{indent}#{key}: #{value.inspect}"
+            end
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/astrails/safe/gpg.rb

@@ -0,0 +1,42 @@
+module Astrails
+  module Safe
+    class Gpg < Pipe
+
+      def compressed?
+        active? || @parent.compressed?
+      end
+
+      protected
+
+      def pipe
+        if key
+          rise RuntimeError, "can't use both gpg password and pubkey" if password
+          "|gpg -e -r #{key}"
+        elsif password
+          "|gpg -c --passphrase-file #{gpg_password_file(password)}"
+        end
+      end
+
+      def extension
+        ".gpg" if active?
+      end
+
+      def active?
+        password || key
+      end
+
+      def password
+        @password ||= config[:gpg, :password]
+      end
+
+      def key
+        @key ||= config[:gpg, :key]
+      end
+
+      def gpg_password_file(pass)
+        return "TEMP_GENERATED_FILENAME" if $DRY_RUN
+        Astrails::Safe::TmpFile.create("gpg-pass") { |file| file.write(pass) }
+      end
+    end
+  end
+end

data/lib/astrails/safe/gzip.rb

@@ -0,0 +1,25 @@
+module Astrails
+  module Safe
+    class Gzip < Pipe
+
+      def compressed?
+        true
+      end
+
+      protected
+
+      def pipe
+        "|gzip" if active?
+      end
+
+      def extension
+        ".gz" if active?
+      end
+
+      def active?
+        !@parent.compressed?
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/local.rb

@@ -0,0 +1,54 @@
+module Astrails
+  module Safe
+    class Local < Sink
+
+      def open(&block)
+        return @parent.open(&block) unless active?
+        run
+        File.open(path, &block) unless $DRY_RUN
+      end
+
+      protected
+
+      def active?
+        # S3 can't upload from pipe. it needs to know file size, so we must pass through :local
+        # will change once we add SSH sink
+        true
+      end
+
+      def prefix
+        @prefix ||= File.expand_path(expand(@config[:local, :path] || raise(RuntimeError, "missing :local/:path in configuration")))
+      end
+
+      def command
+        "#{@parent.command} > #{path}"
+      end
+
+      def save
+        puts "command: #{command}" if $_VERBOSE
+        unless $DRY_RUN
+          FileUtils.mkdir_p(prefix) unless File.directory?(prefix)
+          system command
+        end
+      end
+
+      def cleanup
+        return unless keep = @config[:keep, :local]
+
+        base = File.basename(filename).split(".").first
+
+        pattern = File.join(prefix, "#{base}*")
+        puts "listing files #{pattern.inspect}" if $_VERBOSE
+        files = Dir[pattern] .
+          select{|f| File.file?(f)} .
+          sort
+
+        cleanup_with_limit(files, keep) do |f|
+          puts "removing local file #{f}" if $DRY_RUN || $_VERBOSE
+          File.unlink(f) unless $DRY_RUN
+        end
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/mysqldump.rb

@@ -0,0 +1,31 @@
+module Astrails
+  module Safe
+    class Mysqldump < Source
+
+      def command
+        @commanbd ||= "mysqldump --defaults-extra-file=#{mysql_password_file} #{@config[:options]} #{mysql_skip_tables} #{@id}"
+      end
+
+      def extension; '.sql'; end
+
+      protected
+
+      def mysql_password_file
+        Astrails::Safe::TmpFile.create("mysqldump") do |file|
+          file.puts "[mysqldump]"
+          %w/user password socket host port/.each do |k|
+            v = @config[k]
+            file.puts "#{k} = #{v}" if v
+          end
+        end
+      end
+
+      def mysql_skip_tables
+        if skip_tables = @config[:skip_tables]
+          [*skip_tables].map { |t| "--ignore-table=#{@id}.#{t}" } * " "
+        end
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/pgdump.rb

@@ -0,0 +1,43 @@
+module Astrails
+  module Safe
+    class Pgdump < Source
+
+      def command
+        @commanbd ||= "pg_dump #{postgres_options} #{postgres_username} #{postgres_password} #{postgres_host} #{postgres_port} #{@id}"
+#        @commanbd ||= "pg_dump -U #{@config["user"]} #{@config[:option]} #{@config["database"]} -f #{filename}"
+#        @commanbd ||= "mysqldump --defaults-extra-file=#{mysql_password_file} #{@config[:options]} #{mysql_skip_tables} #{@id}"
+      end
+
+      def extension; '.sql'; end
+
+      protected
+
+      def postgres_options
+        @config[:options]
+      end
+
+      def postgres_host
+        @config["host"] ? "--host='#{@config["port"]}'" : ""
+      end
+
+      def postgres_port
+        @config["port"] ? "--port='#{@config["port"]}'" : ""
+      end
+
+      def postgres_username
+        @config["user"] ? "--username='#{@config["user"]}'" : ""
+      end
+
+      def postgres_password
+        `export PGPASSWORD=#{@config["password"]}` if @config["password"]
+      end
+
+      def postgres_skip_tables
+        if skip_tables = @config[:skip_tables]
+          [*skip_tables].map { |t| "--exclude-table=#{@id}.#{t}" } * " "
+        end
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/pipe.rb

@@ -0,0 +1,15 @@
+module Astrails
+  module Safe
+    class Pipe < Stream
+
+      def command
+        "#{@parent.command}#{pipe}"
+      end
+
+      def filename
+        "#{@parent.filename}#{extension}"
+      end
+
+    end
+  end
+end