astrails-safe 0.0.7 → 0.0.8

data/lib/astrails/safe.rb

@@ -0,0 +1,26 @@
+require 'astrails/safe/tmp_file'
+require 'astrails/safe/config/node'
+require 'astrails/safe/config/builder'
+require 'astrails/safe/stream'
+require 'astrails/safe/engine'
+require 'astrails/safe/mysqldump'
+require 'astrails/safe/archive'
+
+module Astrails
+  module Safe
+    ROOT = File.join(File.dirname(__FILE__), "..", "..")
+
+    def timestamp
+      @timestamp ||= Time.now.strftime("%y%m%d-%H%M")
+    end
+
+    def safe(&block)
+      config = Config::Node.new(&block)
+      #config.dump
+
+      Astrails::Safe::Mysqldump.run(config[:mysqldump, :databases], timestamp)
+      Astrails::Safe::Archive.run(config[:tar, :archives], timestamp)
+    end
+  end
+end
+

data/lib/astrails/safe/archive.rb

@@ -0,0 +1,22 @@
+module Astrails
+  module Safe
+    class Archive < Engine
+
+      def command
+        "tar -cf - #{@config[:options]} #{tar_exclude_files} #{tar_files}"
+      end
+
+      protected
+
+      def tar_exclude_files
+        [*@config[:exclude]].compact.map{|x| "--exclude=#{x}"} * " "
+      end
+
+      def tar_files
+        raise RuntimeError, "missing files for tar" unless @config[:files]
+        [*@config[:files]] * " "
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/config/builder.rb

@@ -0,0 +1,58 @@
+module Astrails
+  module Safe
+    module Config
+      class Builder
+        COLLECTIONS = %w/database archive/
+        ITEMS = %w/s3 key secret bucket path gpg password keep local mysqldump options
+        user socket tar files exclude/
+        NAMES = COLLECTIONS + ITEMS
+        def initialize(node)
+          @node = node
+        end
+
+        # supported args:
+        #   args = [value]
+        #   args = [id, data]
+        #   args = [data]
+        # id/value - simple values, data - hash
+        def method_missing(sym, *args, &block)
+          return super unless NAMES.include?(sym.to_s)
+
+          # do we have id or value?
+          unless args.first.is_a?(Hash)
+            id_or_value = args.shift # nil for args == []
+          end
+
+          # do we have data hash?
+          if data = args.shift
+            die "#{sym}: hash expected: #{data.inspect}" unless data.is_a?(Hash)
+          end
+
+          #puts "#{sym}: args=#{args.inspect}, id_or_value=#{id_or_value}, data=#{data.inspect}, block=#{block.inspect}"
+
+          die "#{sym}: unexpected: #{args.inspect}" unless args.empty?
+          die "#{sym}: missing arguments" unless id_or_value || data || block
+
+          if COLLECTIONS.include?(sym.to_s) && id_or_value
+            data ||= {}
+          end
+
+          if !data && !block
+            # simple value assignment
+            @node[sym] = id_or_value
+
+          elsif id_or_value
+            # collection element with id => create collection node and a subnode in it
+            key = sym.to_s + "s"
+            collection = @node[key] || @node.set(key, {})
+            collection.set(id_or_value, data || {}, &block)
+
+          else
+            # simple subnode
+            @node.set(sym, data || {}, &block)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/astrails/safe/config/node.rb

@@ -0,0 +1,58 @@
+require 'astrails/safe/config/builder'
+module Astrails
+  module Safe
+    module Config
+      class Node
+        def initialize(parent = nil, data = {}, &block)
+          @parent, @data = parent, {}
+          data.each { |k, v| self[k] = v }
+          Builder.new(self).instance_eval(&block) if block
+        end
+
+        # looks for the path from this node DOWN. will not delegate to parent
+        def get(*path)
+          key = path.shift
+          value = @data[key.to_s]
+          return value if value && path.empty?
+
+          value && value.get(*path)
+        end
+
+        # recursive find
+        # starts at the node and continues to the parent
+        def find(*path)
+          get(*path) || @parent && @parent.find(*path)
+        end
+        alias :[] :find
+
+        def set(key, value, &block)
+          @data[key.to_s] =
+            if value.is_a?(Hash)
+              Node.new(self, value, &block)
+            else
+              raise(ArgumentError, "#{key}: no block supported for simple values") if block
+              value
+            end
+        end
+        alias :[]= :set
+
+        def each(&block)
+          @data.each(&block)
+        end
+        include Enumerable
+
+        def dump(indent = "")
+          @data.each do |key, value|
+            if value.is_a?(Node)
+              puts "#{indent}#{key}:"
+              value.dump(indent + "    ")
+            else
+              puts "#{indent}#{key}: #{value.inspect}"
+            end
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/astrails/safe/engine.rb

@@ -0,0 +1,133 @@
+module Astrails
+  module Safe
+    class Engine
+
+      def self.run(config, timestamp)
+        unless config
+          puts "No configuration found for #{kind}"
+          return
+        end
+        config.each do |key, value|
+          new(key, value, timestamp).run
+        end
+      end
+
+      attr_accessor :timestamp
+      attr_reader :id, :config
+      def initialize(id, config, timestamp)
+        @config, @id, @timestamp = config, id, timestamp
+      end
+
+      def run
+        puts "#{kind}: #{@id}" if $_VERBOSE
+
+        stream = Stream.new(@config, command, File.join(path, backup_filename))
+        stream.run # execute backup comand. result is file stream.filename
+
+        # UPLOAD
+        upload(stream.filename)
+
+        # CLEANUP
+        cleanup_s3(stream.filename)
+        cleanup_local(stream.filename)
+      end
+
+      protected
+
+      def kind
+        self.class.name.split('::').last.downcase
+      end
+
+      def backup_filename
+        @backup_filename ||= "#{kind}-#{id}.#{timestamp}.tar"
+      end
+
+      def expand(path)
+        path.gsub(/:kind\b/, kind).gsub(/:id\b/, id)
+      end
+
+      def s3_path
+        @s3_path ||= expand(@config[:s3, :path] || ":kind/:id")
+      end
+
+      def path
+        @path ||= File.expand_path(expand(@config[:path] || raise(RuntimeError, "missing :path in configuration")))
+      end
+
+
+      def upload(filename)
+
+        bucket = @config[:s3, :bucket]
+        key    = @config[:s3, :key]
+        secret = @config[:s3, :secret]
+
+        return unless bucket && key && secret
+
+        upload_path = File.join(s3_path, File.basename(filename))
+
+        puts "Uploading file #{filename} to #{bucket}/#{upload_path}" if $_VERBOSE || $DRY_RUN
+        if $LOCAL
+          puts "skip upload (local operation)"
+        else
+          # needed in cleanup even on dry run
+          AWS::S3::Base.establish_connection!(:access_key_id => key, :secret_access_key => secret, :use_ssl => true)
+
+          unless $DRY_RUN
+            AWS::S3::Bucket.create(bucket)
+            AWS::S3::S3Object.store(s3_path, open(filename), bucket)
+          end
+        end
+        puts "...done" if $_VERBOSE
+      end
+
+      # call block on files to be removed (all except for the LAST 'limit' files
+      def cleanup_files(files, limit, &block)
+        return unless files.size > limit
+
+        to_remove = files[0..(files.size - limit - 1)]
+        to_remove.each(&block)
+      end
+
+      def cleanup_local(filename)
+        return unless keep = @config[:keep, :local]
+
+        dir = File.dirname(filename)
+        base = File.basename(filename).split(".").first
+
+        files = Dir[File.join(dir, "#{base}*")] .
+          select{|f| File.file?(f)} .
+          sort
+
+        cleanup_files(files, keep) do |f|
+          puts "removing local file #{f}" if $DRY_RUN || $_VERBOSE
+          File.unlink(f) unless $DRY_RUN
+        end
+      end
+
+      def cleanup_s3(filename)
+
+        return unless keep = @config[:keep, :s3]
+
+        bucket = @config[:s3, :bucket]
+
+        base = File.basename(filename).split(".").first
+
+        puts "listing files in #{bucket}:#{s3_path}"
+        files = AWS::S3::Bucket.objects(bucket, :prefix => s3_path, :max_keys => keep * 2)
+        puts files.collect(&:key) if $_VERBOSE
+
+        files = files.
+          collect(&:key).
+          select{|f| File.basename(f)[0..(base.length - 1)] == base}.
+          sort
+
+        cleanup_files(files, keep) do |f|
+          puts "removing s3 file #{bucket}:#{f}" if $DRY_RUN || $_VERBOSE
+          AWS::S3::Bucket.find(bucket)[f].delete unless $DRY_RUN
+        end
+      end
+
+    end
+  end
+end
+

data/lib/astrails/safe/mysqldump.rb

@@ -0,0 +1,33 @@
+module Astrails
+  module Safe
+    class Mysqldump < Engine
+
+      def command
+        "mysqldump  --defaults-extra-file=#{password_file} #{@config[:options]} #{mysql_skip_tables} #{@id}"
+      end
+
+      protected
+
+      def password_file
+        Astrails::Safe::TmpFile.create("mysqldump") do |file|
+          file.puts "[mysqldump]"
+          %w/user password socket host port/.each do |k|
+            v = @config[k]
+            file.puts "#{k} = #{v}" if v
+          end
+        end
+      end
+
+      def mysql_skip_tables
+        if skip_tables = @config[:skip_tables]
+          [*skip_tables].map { |t| "--ignore-table=#{@id}.#{t}" } * " "
+        end
+      end
+
+      def mysqldump_extra_options
+        @config[:options] + " " if @config[:options]
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/stream.rb

@@ -0,0 +1,65 @@
+module Astrails
+  module Safe
+    class Stream
+
+      attr_accessor :config, :command, :filename
+      def initialize(config, command, filename)
+        @config, @command, @filename = config, command, filename
+      end
+
+      def run
+        encrypt || compress # use gpg or gzip
+        redirect
+        execute
+      end
+
+      private
+
+      def gpg_password_file(pass)
+        Astrails::Safe::TmpFile.create("gpg-pass") { |file| file.write(pass) }
+      end
+
+      def encrypt
+
+        password = @config[:gpg, :password]
+        key      = @config[:gpg, :key]
+
+        return false unless key || password
+
+        if key
+          rise RuntimeError, "can't use both gpg password and pubkey" if password
+
+          @filename << ".gpg"
+          @command << "|gpg -e -r #{key}"
+        else
+          @filename << ".gpg"
+          unless $DRY_RUN
+            @command << "|gpg -c --passphrase-file #{gpg_password_file(password)}"
+          else
+            @command << "|gpg -c --passphrase-file TEMP_GENERATED_FILENAME"
+          end
+        end
+        true
+      end
+
+      def compress
+        @filename << ".gz"
+        @command << "|gzip"
+      end
+
+      def redirect
+        @command << ">" << @filename
+      end
+
+      def execute
+        dir = File.dirname(filename)
+        FileUtils.mkdir_p(dir) unless File.directory?(dir) || $DRY_RUN
+
+        # EXECUTE
+        puts "Backup command: #{@command}" if $DRY_RUN || $_VERBOSE
+        system @command unless $DRY_RUN
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/tmp_file.rb

@@ -0,0 +1,21 @@
+require 'tmpdir'
+module Astrails
+  module Safe
+    module TmpFile
+      @KEEP_FILES = []
+      TMPDIR = Dir.mktmpdir
+
+      def self.create(name)
+        # create temp directory
+
+        file = Tempfile.new(name, TMPDIR)
+
+        yield file
+
+        file.close
+        @KEEP_FILES << file # so that it will not get gcollected and removed from filesystem until the end
+        file.path
+      end
+    end
+  end
+end

data/safe.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name     = "safe"
-  s.version  = "0.0.7"
+  s.version  = "0.0.8"
   s.date     = "2009-03-15"
   s.summary  = "Astrails Safe"
   s.email    = "we@astrails.com"
@@ -9,9 +9,18 @@ Gem::Specification.new do |s|
   s.has_rdoc = false
   s.authors  = ["Astrails Ltd."]
   s.files    = files = %w(
-    bin/astrails-safe
-    safe.gemspec
-  ) + Dir['lib/**/*'] + Dir['template/**/*']
+  bin/astrails-safe
+  lib/astrails/safe.rb
+  lib/astrails/safe/mysqldump.rb
+  lib/astrails/safe/stream.rb
+  lib/astrails/safe/config/builder.rb
+  lib/astrails/safe/config/node.rb
+  lib/astrails/safe/engine.rb
+  lib/astrails/safe/archive.rb
+  lib/astrails/safe/tmp_file.rb
+  templates/script.rb
+  safe.gemspec
+  )
   s.executables = files.grep(/^bin/).map {|x| x.gsub(/^bin\//, "")}
 
   s.test_files = []

data/templates/script.rb

@@ -0,0 +1,106 @@
+safe do
+  # global path
+  # supported substitutions:
+  #   :kind -> backup 'engine' kind, e.g. "mysqldump" or "archive"
+  #   :id -> backup 'id', e.g. "blog", "production", etc.
+  # you can set separate :path for all backups (or once globally here)
+  path "/backup/:kind/:id"
+
+  ## uncomment to enable uploads to Amazon S3
+  ## Amazon S3 auth (optional)
+  # s3 do
+  #   key YOUR_S3_KEY
+  #   secret YOUR_S3_SECRET
+  #   bucket S3_BUCKET
+  # end
+
+  ## alternative style:
+  # s3 :key => YOUR_S3_KEY, :secret => YOUR_S3_SECRET, :bucket => S3_BUCKET
+
+  ## uncomment to enable GPG encryption.
+  ## Note: you can use public 'key' or symmetric password but not both!
+  # gpg do
+  #   # key "backup@astrails.com"
+  #   password "astrails"
+  # end
+
+  ## uncomment to enable backup rotation. keep only given number of latest
+  ## backups. remove the rest
+  # keep do
+  #   local 4 # keep 4 local backups
+  #   s3 20 # keep 20 S3 backups
+  # end
+
+  # backup mysql databases with mysqldump
+  mysqldump do
+    # you can override any setting from parent in a child:
+    path "/backup/mysql/:id"
+
+    options "-ceKq --single-transaction --create-options"
+
+    user "astrails"
+    password ""
+    # host "localhost"
+    # port 3306
+    socket "/var/run/mysqld/mysqld.sock"
+
+    # database is a 'collection' element. it must have a hash or block parameter
+    # it will be 'collected' in a 'databases', with database id (1st arg) used as hash key
+    # the following code will create mysqldump/databases/blog and mysqldump/databases/mysql ocnfiguration 'nodes'
+
+    # backup database with default values
+    # database :blog
+
+    # backup overriding some values
+    # database :production do
+    #   # you can override 'partially'
+    #   keep :local => 3
+    #   # keep/local is 3, and keep/s3 is 20 (from parent)
+
+    #   # local override for gpg password
+    #   gpg do
+    #     password "custom-production-pass"
+    #   end
+
+    #   skip_tables [:logger_exceptions, :request_logs] # skip those tables during backup
+    # end
+
+  end
+
+
+  tar do
+    path "/backup/archives"
+
+    # 'archive' is a collection item, just like 'database'
+    # archive "git-repositories" do
+    #   # files and directories to backup
+    #   files "/home/git/repositories"
+    # end
+
+    # archive "etc-files" do
+    #   files "/etc"
+    #   # exlude those files/directories
+    #   exclude "/etc/puppet/other"
+    # end
+
+    # archive "dot-configs" do
+    #   files "/home/*/.[^.]*"
+    # end
+
+    # archive "blog" do
+    #   files "/var/www/blog.astrails.com/"
+    #   # specify multiple files/directories as array
+    #   exclude ["/var/www/blog.astrails.com/log", "/var/www/blog.astrails.com/tmp"]
+    # end
+
+    # archive "site" do
+    #   files "/var/www/astrails.com/"
+    #   exclude ["/var/www/astrails.com/log", "/var/www/astrails.com/tmp"]
+    # end
+
+    # archive :misc do
+    #   files [ "/backup/*.rb" ]
+    # end
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: astrails-safe
 version: !ruby/object:Gem::Version 
-  version: 0.0.7
+  version: 0.0.8
 platform: ruby
 authors: 
 - Astrails Ltd.
@@ -32,6 +32,15 @@ extra_rdoc_files: []
 
 files: 
 - bin/astrails-safe
+- lib/astrails/safe.rb
+- lib/astrails/safe/mysqldump.rb
+- lib/astrails/safe/stream.rb
+- lib/astrails/safe/config/builder.rb
+- lib/astrails/safe/config/node.rb
+- lib/astrails/safe/engine.rb
+- lib/astrails/safe/archive.rb
+- lib/astrails/safe/tmp_file.rb
+- templates/script.rb
 - safe.gemspec
 has_rdoc: false
 homepage: http://github.com/astrails/safe