astrails-safe 0.1.0 → 0.1.1

data/README

@@ -0,0 +1,13 @@
+Simple mysql and filesystem backups with S3 support
+
+Usage:
+   astrails-safe [OPTIONS] CONFIG_FILE
+Options:
+  -h, --help           This help screen
+  -v, --verbose        be verbose, duh!
+  -n, --dry-run        just pretend, don't do anything.
+  -L, --local          skip S3
+
+Note: CONFIG_FILE will be created from template if missing
+
+See template for configuration examples

data/lib/astrails/safe.rb

@@ -1,12 +1,24 @@
 require 'extensions/mktmpdir'
 require 'astrails/safe/tmp_file'
+
 require 'astrails/safe/config/node'
 require 'astrails/safe/config/builder'
+
 require 'astrails/safe/stream'
-require 'astrails/safe/engine'
+
+require 'astrails/safe/source'
 require 'astrails/safe/mysqldump'
 require 'astrails/safe/archive'
 
+require 'astrails/safe/pipe'
+require 'astrails/safe/gpg'
+require 'astrails/safe/gzip'
+
+require 'astrails/safe/sink'
+require 'astrails/safe/local'
+require 'astrails/safe/s3'
+
+
 module Astrails
   module Safe
     ROOT = File.join(File.dirname(__FILE__), "..", "..")
@@ -19,8 +31,9 @@ module Astrails
       config = Config::Node.new(&block)
       #config.dump
 
-      Astrails::Safe::Mysqldump.run(config[:mysqldump, :databases], timestamp)
-      Astrails::Safe::Archive.run(config[:tar, :archives], timestamp)
+      Astrails::Safe::Mysqldump.run(config[:mysqldump, :databases])
+      Astrails::Safe::Archive.run(config[:tar, :archives])
+
       Astrails::Safe::TmpFile.cleanup
     end
   end

data/lib/astrails/safe/archive.rb

@@ -1,12 +1,12 @@
 module Astrails
   module Safe
-    class Archive < Engine
+    class Archive < Source
 
       def command
         "tar -cf - #{@config[:options]} #{tar_exclude_files} #{tar_files}"
       end
 
-      def extension; 'tar'; end
+      def extension; '.tar'; end
 
       protected
 

data/lib/astrails/safe/config/builder.rb

@@ -3,8 +3,8 @@ module Astrails
     module Config
       class Builder
         COLLECTIONS = %w/database archive/
-        ITEMS = %w/s3 key secret bucket path prefix gpg password keep local mysqldump options
-        user socket tar files exclude/
+        ITEMS = %w/s3 key secret bucket path gpg password keep local mysqldump options
+        user socket tar files exclude filename/
         NAMES = COLLECTIONS + ITEMS
         def initialize(node)
           @node = node

data/lib/astrails/safe/config/node.rb

@@ -3,6 +3,7 @@ module Astrails
   module Safe
     module Config
       class Node
+        attr_reader :parent
         def initialize(parent = nil, data = {}, &block)
           @parent, @data = parent, {}
           data.each { |k, v| self[k] = v }
@@ -41,6 +42,14 @@ module Astrails
         end
         include Enumerable
 
+        def to_hash
+          @data.keys.inject({}) do |res, key|
+            value = @data[key]
+            res[key] = value.is_a?(Node) ? value.to_hash : value
+            res
+          end
+        end
+
         def dump(indent = "")
           @data.each do |key, value|
             if value.is_a?(Node)

data/lib/astrails/safe/gpg.rb

@@ -0,0 +1,42 @@
+module Astrails
+  module Safe
+    class Gpg < Pipe
+
+      def compressed?
+        active? || @parent.compressed?
+      end
+
+      protected
+
+      def pipe
+        if key
+          rise RuntimeError, "can't use both gpg password and pubkey" if password
+          "|gpg -e -r #{key}"
+        elsif password
+          "|gpg -c --passphrase-file #{gpg_password_file(password)}"
+        end
+      end
+
+      def extension
+        ".gpg" if active?
+      end
+
+      def active?
+        password || key
+      end
+
+      def password
+        @password ||= config[:gpg, :password]
+      end
+
+      def key
+        @key ||= config[:gpg, :key]
+      end
+
+      def gpg_password_file(pass)
+        return "TEMP_GENERATED_FILENAME" if $DRY_RUN
+        Astrails::Safe::TmpFile.create("gpg-pass") { |file| file.write(pass) }
+      end
+    end
+  end
+end

data/lib/astrails/safe/gzip.rb

@@ -0,0 +1,25 @@
+module Astrails
+  module Safe
+    class Gzip < Pipe
+
+      def compressed?
+        true
+      end
+
+      protected
+
+      def pipe
+        "|gzip" if active?
+      end
+
+      def extension
+        ".gz" if active?
+      end
+
+      def active?
+        !@parent.compressed?
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/local.rb

@@ -0,0 +1,54 @@
+module Astrails
+  module Safe
+    class Local < Sink
+
+      def open(&block)
+        return @parent.open(&block) unless active?
+        run
+        File.open(path, &block) unless $DRY_RUN
+      end
+
+      protected
+
+      def active?
+        # S3 can't upload from pipe. it needs to know file size, so we must pass through :local
+        # will change once we add SSH sink
+        true
+      end
+
+      def prefix
+        @prefix ||= File.expand_path(expand(@config[:local, :path] || raise(RuntimeError, "missing :local/:path in configuration")))
+      end
+
+      def command
+        "#{@parent.command} > #{path}"
+      end
+
+      def save
+        puts "command: #{command}" if $_VERBOSE
+        unless $DRY_RUN
+          FileUtils.mkdir_p(prefix) unless File.directory?(prefix)
+          system command
+        end
+      end
+
+      def cleanup
+        return unless keep = @config[:keep, :local]
+
+        base = File.basename(filename).split(".").first
+
+        pattern = File.join(prefix, "#{base}*")
+        puts "listing files #{pattern.inspect}" if $_VERBOSE
+        files = Dir[pattern] .
+          select{|f| File.file?(f)} .
+          sort
+
+        cleanup_with_limit(files, keep) do |f|
+          puts "removing local file #{f}" if $DRY_RUN || $_VERBOSE
+          File.unlink(f) unless $DRY_RUN
+        end
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/mysqldump.rb

@@ -1,16 +1,16 @@
 module Astrails
   module Safe
-    class Mysqldump < Engine
+    class Mysqldump < Source
 
       def command
-        "mysqldump  --defaults-extra-file=#{password_file} #{@config[:options]} #{mysql_skip_tables} #{@id}"
+        @commanbd ||= "mysqldump --defaults-extra-file=#{mysql_password_file} #{@config[:options]} #{mysql_skip_tables} #{@id}"
       end
 
-      def extension; 'sql'; end
+      def extension; '.sql'; end
 
       protected
 
-      def password_file
+      def mysql_password_file
         Astrails::Safe::TmpFile.create("mysqldump") do |file|
           file.puts "[mysqldump]"
           %w/user password socket host port/.each do |k|
@@ -26,10 +26,6 @@ module Astrails
         end
       end
 
-      def mysqldump_extra_options
-        @config[:options] + " " if @config[:options]
-      end
-
     end
   end
 end

data/lib/astrails/safe/pipe.rb

@@ -0,0 +1,15 @@
+module Astrails
+  module Safe
+    class Pipe < Stream
+
+      def command
+        "#{@parent.command}#{pipe}"
+      end
+
+      def filename
+        "#{@parent.filename}#{extension}"
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/s3.rb

@@ -0,0 +1,68 @@
+module Astrails
+  module Safe
+    class S3 < Sink
+
+      protected
+
+      def active?
+        bucket && key && secret
+      end
+
+      def prefix
+        @prefix ||= expand(config[:s3, :path] || expand(config[:local, :path] || ":kind/:id"))
+      end
+
+      def save
+        # needed in cleanup even on dry run
+        AWS::S3::Base.establish_connection!(:access_key_id => key, :secret_access_key => secret, :use_ssl => true) unless $LOCAL
+
+        file = @parent.open
+        puts "Uploading #{bucket}:#{path}" if $_VERBOSE || $DRY_RUN
+        unless $DRY_RUN || $LOCAL
+          AWS::S3::Bucket.create(bucket)
+          AWS::S3::S3Object.store(path, file, bucket)
+          puts "...done" if $_VERBOSE
+        end
+        file.close if file
+
+      end
+
+      def cleanup
+
+        return if $LOCAL
+
+        return unless keep = @config[:keep, :s3]
+
+        bucket = @config[:s3, :bucket]
+
+        base = File.basename(filename).split(".").first
+
+        puts "listing files in #{bucket}:#{prefix}/#{base}"
+        files = AWS::S3::Bucket.objects(bucket, :prefix => "#{prefix}/#{base}", :max_keys => keep * 2)
+        puts files.collect {|x| x.key} if $_VERBOSE
+
+        files = files.
+          collect {|x| x.key}.
+          sort
+
+        cleanup_with_limit(files, keep) do |f|
+          puts "removing s3 file #{bucket}:#{f}" if $DRY_RUN || $_VERBOSE
+          AWS::S3::Bucket.find(bucket)[f].delete unless $DRY_RUN || $LOCAL
+        end
+      end
+
+      def bucket
+        config[:s3, :bucket]
+      end
+
+      def key
+        config[:s3, :key]
+      end
+
+      def secret
+        config[:s3, :secret]
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/sink.rb

@@ -0,0 +1,33 @@
+module Astrails
+  module Safe
+    class Sink < Stream
+
+      def run
+        if active?
+          save
+          cleanup
+        else
+          @parent.run
+        end
+      end
+
+      protected
+
+      # prefix is defined in subclass
+      def path
+        @path ||= File.join(prefix, filename)
+      end
+
+      # call block on files to be removed (all except for the LAST 'limit' files
+      def cleanup_with_limit(files, limit, &block)
+        return unless files.size > limit
+
+        to_remove = files[0..(files.size - limit - 1)]
+        # TODO: validate here
+        to_remove.each(&block)
+      end
+
+    end
+  end
+end
+

data/lib/astrails/safe/source.rb

@@ -0,0 +1,31 @@
+module Astrails
+  module Safe
+    class Source < Stream
+
+      def initialize(id, config)
+        @id, @config = id, config
+      end
+
+      def filename
+        @filename ||= expand(":kind-:id.:timestamp#{extension}")
+      end
+
+      # process each config key as source (with full pipe)
+      def self.run(config)
+        unless config
+          puts "No configuration found for #{kind}"
+          return
+        end
+
+        config.each do |key, value|
+          stream = [Gpg, Gzip, Local, S3].inject(new(key, value)) do |res, klass|
+            klass.new(res)
+          end
+          stream.run
+        end
+      end
+
+    end
+  end
+end
+

data/lib/astrails/safe/stream.rb

@@ -2,64 +2,44 @@ module Astrails
   module Safe
     class Stream
 
-      attr_accessor :config, :command, :filename
-      def initialize(config, command, filename)
-        @config, @command, @filename = config, command, filename
+      def initialize(parent)
+        @parent = parent
       end
 
-      def run
-        encrypt || compress # use gpg or gzip
-        redirect
-        execute
+      def id
+        @id ||= @parent.id
       end
 
-      private
-
-      def gpg_password_file(pass)
-        Astrails::Safe::TmpFile.create("gpg-pass") { |file| file.write(pass) }
+      def config
+        @config ||= @parent.config
       end
 
-      def encrypt
-
-        password = @config[:gpg, :password]
-        key      = @config[:gpg, :key]
-
-        return false unless key || password
-
-        if key
-          rise RuntimeError, "can't use both gpg password and pubkey" if password
-
-          @filename << ".gpg"
-          @command << "|gpg -e -r #{key}"
-        else
-          @filename << ".gpg"
-          unless $DRY_RUN
-            @command << "|gpg -c --passphrase-file #{gpg_password_file(password)}"
-          else
-            @command << "|gpg -c --passphrase-file TEMP_GENERATED_FILENAME"
-          end
-        end
-        true
+      def filename
+        @parent.filename
       end
 
-      def compress
-        @filename << ".gz"
-        @command << "|gzip"
+      def compressed?
+        @parent && @parent.compressed?
       end
 
-      def redirect
-        @command << ">" << @filename
+      protected
+
+      def name
+        self.class.name.split('::').last.downcase
       end
 
-      def execute
-        dir = File.dirname(filename)
-        FileUtils.mkdir_p(dir) unless File.directory?(dir) || $DRY_RUN
+      def kind
+        @parent ? @parent.kind : name
+      end
 
-        # EXECUTE
-        puts "Backup command: #{@command}" if $DRY_RUN || $_VERBOSE
-        system @command unless $DRY_RUN
+      def expand(path)
+        path .
+          gsub(/:kind\b/, kind) .
+          gsub(/:id\b/, id) .
+          gsub(/:timestamp\b/, timestamp)
       end
 
     end
   end
 end
+

data/safe.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name     = "safe"
-  s.version  = "0.1.0"
+  s.version  = "0.1.1"
   s.date     = "2009-03-15"
   s.summary  = "Astrails Safe"
   s.email    = "we@astrails.com"
@@ -9,16 +9,23 @@ Gem::Specification.new do |s|
   s.has_rdoc = false
   s.authors  = ["Astrails Ltd."]
   s.files    = files = %w(
+  README
   bin/astrails-safe
+  lib/extensions/mktmpdir.rb
   lib/astrails/safe.rb
+  lib/astrails/safe/s3.rb
+  lib/astrails/safe/gpg.rb
   lib/astrails/safe/mysqldump.rb
   lib/astrails/safe/stream.rb
   lib/astrails/safe/config/builder.rb
   lib/astrails/safe/config/node.rb
-  lib/astrails/safe/engine.rb
+  lib/astrails/safe/sink.rb
+  lib/astrails/safe/pipe.rb
+  lib/astrails/safe/source.rb
   lib/astrails/safe/archive.rb
+  lib/astrails/safe/local.rb
   lib/astrails/safe/tmp_file.rb
-  lib/extensions/mktmpdir.rb
+  lib/astrails/safe/gzip.rb
   templates/script.rb
   safe.gemspec
   )

data/templates/script.rb

@@ -1,21 +1,24 @@
 safe do
-  # backup file path (full, including filename)
-  # Note: do not include .tar, .sql, .gz or .pgp, it will be added automatically
+
+  # backup file path (not including filename)
   # supported substitutions:
   #   :kind -> backup 'engine' kind, e.g. "mysqldump" or "archive"
   #   :id -> backup 'id', e.g. "blog", "production", etc.
   #   :timestamp -> current run timestamp (same for all the backups in the same 'run')
   # you can set separate :path for all backups (or once globally here)
-  path "/backup/:kind/:id-:timestamp"
+  local do
+    path "/backup/:kind/"
+  end
 
   ## uncomment to enable uploads to Amazon S3
   ## Amazon S3 auth (optional)
+  ## don't forget to add :s3 to the 'store' list
   # s3 do
   #   key YOUR_S3_KEY
   #   secret YOUR_S3_SECRET
   #   bucket S3_BUCKET
-  #   # prefix for uploads to S3. supports same substitution like :path
-  #   prefix ":kind/:id" # this is default
+  #   # path for uploads to S3. supports same substitution like :local/:path
+  #   path ":kind/" # this is default
   # end
 
   ## alternative style:

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: astrails-safe
 version: !ruby/object:Gem::Version 
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors: 
 - Astrails Ltd.
@@ -31,16 +31,23 @@ extensions: []
 extra_rdoc_files: []
 
 files: 
+- README
 - bin/astrails-safe
+- lib/extensions/mktmpdir.rb
 - lib/astrails/safe.rb
+- lib/astrails/safe/s3.rb
+- lib/astrails/safe/gpg.rb
 - lib/astrails/safe/mysqldump.rb
 - lib/astrails/safe/stream.rb
 - lib/astrails/safe/config/builder.rb
 - lib/astrails/safe/config/node.rb
-- lib/astrails/safe/engine.rb
+- lib/astrails/safe/sink.rb
+- lib/astrails/safe/pipe.rb
+- lib/astrails/safe/source.rb
 - lib/astrails/safe/archive.rb
+- lib/astrails/safe/local.rb
 - lib/astrails/safe/tmp_file.rb
-- lib/extensions/mktmpdir.rb
+- lib/astrails/safe/gzip.rb
 - templates/script.rb
 - safe.gemspec
 has_rdoc: false

data/lib/astrails/safe/engine.rb

@@ -1,134 +0,0 @@
-module Astrails
-  module Safe
-    class Engine
-
-      def self.run(config, timestamp)
-        unless config
-          puts "No configuration found for #{kind}"
-          return
-        end
-        config.each do |key, value|
-          new(key, value, timestamp).run
-        end
-      end
-
-      attr_accessor :timestamp
-      attr_reader :id, :config
-      def initialize(id, config, timestamp)
-        @config, @id, @timestamp = config, id, timestamp
-      end
-
-      def run
-        puts "#{kind}: #{@id}" if $_VERBOSE
-
-        stream = Stream.new(@config, command, backup_filepath)
-        stream.run # execute backup comand. result is file stream.filename
-
-        # UPLOAD
-        upload(stream.filename)
-
-        # CLEANUP
-        cleanup_s3(stream.filename)
-        cleanup_local(stream.filename)
-      end
-
-      protected
-
-      def self.kind
-        name.split('::').last.downcase
-      end
-      def kind
-        self.class.kind
-      end
-
-      def expand(path)
-        path .
-          gsub(/:kind\b/, kind) .
-          gsub(/:id\b/, id) .
-          gsub(/:timestamp\b/, timestamp)
-      end
-
-      def s3_prefix
-        @s3_prefix ||= expand(@config[:s3, :prefix] || ":kind/:id")
-      end
-
-      def backup_filepath
-        @backup_filepath ||= File.expand_path(expand(@config[:path] || raise(RuntimeError, "missing :path in configuration"))) << "." << extension
-      end
-
-
-      def upload(filename)
-
-        bucket = @config[:s3, :bucket]
-        key    = @config[:s3, :key]
-        secret = @config[:s3, :secret]
-
-        return unless bucket && key && secret
-
-        upload_path = File.join(s3_prefix, File.basename(filename))
-
-        puts "Uploading file #{filename} to #{bucket}/#{upload_path}" if $_VERBOSE || $DRY_RUN
-        if $LOCAL
-          puts "skip upload (local operation)"
-        else
-          # needed in cleanup even on dry run
-          AWS::S3::Base.establish_connection!(:access_key_id => key, :secret_access_key => secret, :use_ssl => true)
-
-          unless $DRY_RUN
-            AWS::S3::Bucket.create(bucket)
-            AWS::S3::S3Object.store(upload_path, open(filename), bucket)
-          end
-        end
-        puts "...done" if $_VERBOSE
-      end
-
-      # call block on files to be removed (all except for the LAST 'limit' files
-      def cleanup_files(files, limit, &block)
-        return unless files.size > limit
-
-        to_remove = files[0..(files.size - limit - 1)]
-        to_remove.each(&block)
-      end
-
-      def cleanup_local(filename)
-        return unless keep = @config[:keep, :local]
-
-        dir = File.dirname(filename)
-        base = File.basename(filename).split(".").first
-
-        files = Dir[File.join(dir, "#{base}*")] .
-          select{|f| File.file?(f)} .
-          sort
-
-        cleanup_files(files, keep) do |f|
-          puts "removing local file #{f}" if $DRY_RUN || $_VERBOSE
-          File.unlink(f) unless $DRY_RUN
-        end
-      end
-
-      def cleanup_s3(filename)
-
-        return unless keep = @config[:keep, :s3]
-
-        bucket = @config[:s3, :bucket]
-
-        base = File.basename(filename).split(".").first
-
-        puts "listing files in #{bucket}:#{s3_prefix}"
-        files = AWS::S3::Bucket.objects(bucket, :prefix => "#{s3_prefix}/#{base}", :max_keys => keep * 2)
-        puts files.collect {|x| x.key} if $_VERBOSE
-
-        files = files.
-          collect {|x| x.key}.
-          sort
-
-        cleanup_files(files, keep) do |f|
-          puts "removing s3 file #{bucket}:#{f}" if $DRY_RUN || $_VERBOSE
-          AWS::S3::Bucket.find(bucket)[f].delete unless $DRY_RUN
-        end
-      end
-
-    end
-  end
-end
-