markmansour-safe 0.1.7

data/lib/astrails/safe/s3.rb

@@ -0,0 +1,68 @@
+module Astrails
+  module Safe
+    class S3 < Sink
+
+      protected
+
+      def active?
+        bucket && key && secret
+      end
+
+      def prefix
+        @prefix ||= expand(config[:s3, :path] || expand(config[:local, :path] || ":kind/:id"))
+      end
+
+      def save
+        # needed in cleanup even on dry run
+        AWS::S3::Base.establish_connection!(:access_key_id => key, :secret_access_key => secret, :use_ssl => true) unless $LOCAL
+
+        file = @parent.open
+        puts "Uploading #{bucket}:#{path}" if $_VERBOSE || $DRY_RUN
+        unless $DRY_RUN || $LOCAL
+          AWS::S3::Bucket.create(bucket)
+          AWS::S3::S3Object.store(path, file, bucket)
+          puts "...done" if $_VERBOSE
+        end
+        file.close if file
+
+      end
+
+      def cleanup
+
+        return if $LOCAL
+
+        return unless keep = @config[:keep, :s3]
+
+        bucket = @config[:s3, :bucket]
+
+        base = File.basename(filename).split(".").first
+
+        puts "listing files in #{bucket}:#{prefix}/#{base}"
+        files = AWS::S3::Bucket.objects(bucket, :prefix => "#{prefix}/#{base}", :max_keys => keep * 2)
+        puts files.collect {|x| x.key} if $_VERBOSE
+
+        files = files.
+          collect {|x| x.key}.
+          sort
+
+        cleanup_with_limit(files, keep) do |f|
+          puts "removing s3 file #{bucket}:#{f}" if $DRY_RUN || $_VERBOSE
+          AWS::S3::Bucket.find(bucket)[f].delete unless $DRY_RUN || $LOCAL
+        end
+      end
+
+      def bucket
+        config[:s3, :bucket]
+      end
+
+      def key
+        config[:s3, :key]
+      end
+
+      def secret
+        config[:s3, :secret]
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/sink.rb

@@ -0,0 +1,33 @@
+module Astrails
+  module Safe
+    class Sink < Stream
+
+      def run
+        if active?
+          save
+          cleanup
+        else
+          @parent.run
+        end
+      end
+
+      protected
+
+      # prefix is defined in subclass
+      def path
+        @path ||= File.join(prefix, filename)
+      end
+
+      # call block on files to be removed (all except for the LAST 'limit' files
+      def cleanup_with_limit(files, limit, &block)
+        return unless files.size > limit
+
+        to_remove = files[0..(files.size - limit - 1)]
+        # TODO: validate here
+        to_remove.each(&block)
+      end
+
+    end
+  end
+end
+

data/lib/astrails/safe/source.rb

@@ -0,0 +1,31 @@
+module Astrails
+  module Safe
+    class Source < Stream
+
+      def initialize(id, config)
+        @id, @config = id, config
+      end
+
+      def filename
+        @filename ||= expand(":kind-:id.:timestamp#{extension}")
+      end
+
+      # process each config key as source (with full pipe)
+      def self.run(config)
+        unless config
+          puts "No configuration found for #{human_name}"
+          return
+        end
+
+        config.each do |key, value|
+          stream = [Gpg, Gzip, Local, S3].inject(new(key, value)) do |res, klass|
+            klass.new(res)
+          end
+          stream.run
+        end
+      end
+
+    end
+  end
+end
+

data/lib/astrails/safe/stream.rb

@@ -0,0 +1,45 @@
+module Astrails
+  module Safe
+    class Stream
+
+      def initialize(parent)
+        @parent = parent
+      end
+
+      def id
+        @id ||= @parent.id
+      end
+
+      def config
+        @config ||= @parent.config
+      end
+
+      def filename
+        @parent.filename
+      end
+
+      def compressed?
+        @parent && @parent.compressed?
+      end
+
+      protected
+
+      def self.human_name
+        name.split('::').last.downcase
+      end
+
+      def kind
+        @parent ? @parent.kind : self.class.human_name
+      end
+
+      def expand(path)
+        path .
+          gsub(/:kind\b/, kind) .
+          gsub(/:id\b/, id) .
+          gsub(/:timestamp\b/, timestamp)
+      end
+
+    end
+  end
+end
+

data/lib/astrails/safe/tmp_file.rb

@@ -0,0 +1,25 @@
+require 'tmpdir'
+module Astrails
+  module Safe
+    module TmpFile
+      @KEEP_FILES = []
+      TMPDIR = Dir.mktmpdir
+
+      def self.cleanup
+        FileUtils.remove_entry_secure TMPDIR
+      end
+
+      def self.create(name)
+        # create temp directory
+
+        file = Tempfile.new(name, TMPDIR)
+
+        yield file
+
+        file.close
+        @KEEP_FILES << file # so that it will not get gcollected and removed from filesystem until the end
+        file.path
+      end
+    end
+  end
+end

data/lib/extensions/mktmpdir.rb

@@ -0,0 +1,45 @@
+require 'tmpdir'
+
+unless Dir.respond_to?(:mktmpdir)
+  # backward compat for 1.8.6
+  class Dir
+    def Dir.mktmpdir(prefix_suffix=nil, tmpdir=nil)
+      case prefix_suffix
+      when nil
+        prefix = "d"
+        suffix = ""
+      when String
+        prefix = prefix_suffix
+        suffix = ""
+      when Array
+        prefix = prefix_suffix[0]
+        suffix = prefix_suffix[1]
+      else
+        raise ArgumentError, "unexpected prefix_suffix: #{prefix_suffix.inspect}"
+      end
+      tmpdir ||= Dir.tmpdir
+      t = Time.now.strftime("%Y%m%d")
+      n = nil
+      begin
+        path = "#{tmpdir}/#{prefix}#{t}-#{$$}-#{rand(0x100000000).to_s(36)}"
+        path << "-#{n}" if n
+        path << suffix
+        Dir.mkdir(path, 0700)
+      rescue Errno::EEXIST
+        n ||= 0
+        n += 1
+        retry
+      end
+
+      if block_given?
+        begin
+          yield path
+        ensure
+          FileUtils.remove_entry_secure path
+        end
+      else
+        path
+      end
+    end
+  end
+end

data/templates/script.rb

@@ -0,0 +1,122 @@
+safe do
+
+  # backup file path (not including filename)
+  # supported substitutions:
+  #   :kind -> backup 'engine' kind, e.g. "mysqldump" or "archive"
+  #   :id -> backup 'id', e.g. "blog", "production", etc.
+  #   :timestamp -> current run timestamp (same for all the backups in the same 'run')
+  # you can set separate :path for all backups (or once globally here)
+  local do
+    path "/backup/:kind/"
+  end
+
+  ## uncomment to enable uploads to Amazon S3
+  ## Amazon S3 auth (optional)
+  ## don't forget to add :s3 to the 'store' list
+  # s3 do
+  #   key YOUR_S3_KEY
+  #   secret YOUR_S3_SECRET
+  #   bucket S3_BUCKET
+  #   # path for uploads to S3. supports same substitution like :local/:path
+  #   path ":kind/" # this is default
+  # end
+
+  ## alternative style:
+  # s3 :key => YOUR_S3_KEY, :secret => YOUR_S3_SECRET, :bucket => S3_BUCKET
+
+  ## uncomment to enable GPG encryption.
+  ## Note: you can use public 'key' or symmetric password but not both!
+  # gpg do
+  #   # key "backup@astrails.com"
+  #   password "astrails"
+  # end
+
+  ## uncomment to enable backup rotation. keep only given number of latest
+  ## backups. remove the rest
+  # keep do
+  #   local 4 # keep 4 local backups
+  #   s3 20 # keep 20 S3 backups
+  # end
+
+  # backup mysql databases with mysqldump
+  mysqldump do
+    # you can override any setting from parent in a child:
+    options "-ceKq --single-transaction --create-options"
+
+    user "astrails"
+    password ""
+    # host "localhost"
+    # port 3306
+    socket "/var/run/mysqld/mysqld.sock"
+
+    # database is a 'collection' element. it must have a hash or block parameter
+    # it will be 'collected' in a 'databases', with database id (1st arg) used as hash key
+    # the following code will create mysqldump/databases/blog and mysqldump/databases/mysql ocnfiguration 'nodes'
+
+    # backup database with default values
+    # database :blog
+
+    # backup overriding some values
+    # database :production do
+    #   # you can override 'partially'
+    #   keep :local => 3
+    #   # keep/local is 3, and keep/s3 is 20 (from parent)
+
+    #   # local override for gpg password
+    #   gpg do
+    #     password "custom-production-pass"
+    #   end
+
+    #   skip_tables [:logger_exceptions, :request_logs] # skip those tables during backup
+    # end
+
+  end
+
+  # # uncomment to enable
+  # # backup PostgreSQL databases with pg_dump
+  # pgdump do
+  #   option "-i -x -O"
+  #
+  #   user "markmansour"
+  #   # password "" - leave this out if you have ident setup
+  #
+  #   # database is a 'collection' element. it must have a hash or block parameter
+  #   # it will be 'collected' in a 'databases', with database id (1st arg) used as hash key
+  #   database :blog
+  #   database :production
+  # end
+
+  tar do
+    # 'archive' is a collection item, just like 'database'
+    # archive "git-repositories" do
+    #   # files and directories to backup
+    #   files "/home/git/repositories"
+    # end
+
+    # archive "etc-files" do
+    #   files "/etc"
+    #   # exlude those files/directories
+    #   exclude "/etc/puppet/other"
+    # end
+
+    # archive "dot-configs" do
+    #   files "/home/*/.[^.]*"
+    # end
+
+    # archive "blog" do
+    #   files "/var/www/blog.astrails.com/"
+    #   # specify multiple files/directories as array
+    #   exclude ["/var/www/blog.astrails.com/log", "/var/www/blog.astrails.com/tmp"]
+    # end
+
+    # archive "site" do
+    #   files "/var/www/astrails.com/"
+    #   exclude ["/var/www/astrails.com/log", "/var/www/astrails.com/tmp"]
+    # end
+
+    # archive :misc do
+    #   files [ "/backup/*.rb" ]
+    # end
+  end
+
+end

metadata

@@ -0,0 +1,94 @@
+--- !ruby/object:Gem::Specification 
+name: markmansour-safe
+version: !ruby/object:Gem::Version 
+  version: 0.1.7
+platform: ruby
+authors: 
+- Astrails Ltd.
+- Mark Mansour
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-04-24 00:00:00 -07:00
+default_executable: astrails-safe
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: aws-s3
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: Simple tool to backup MySQL and PostgreSQL databases and filesystem locally or to Amazon S3 (with optional encryption)
+email: we@astrails.com
+executables: 
+- astrails-safe
+extensions: []
+
+extra_rdoc_files: 
+- README.markdown
+- LICENSE
+files: 
+- README.markdown
+- VERSION.yml
+- bin/astrails-safe
+- examples/example_helper.rb
+- examples/unit
+- examples/unit/config_example.rb
+- examples/unit/stream_example.rb
+- lib/astrails
+- lib/astrails/safe
+- lib/astrails/safe/archive.rb
+- lib/astrails/safe/config
+- lib/astrails/safe/config/builder.rb
+- lib/astrails/safe/config/node.rb
+- lib/astrails/safe/gpg.rb
+- lib/astrails/safe/gzip.rb
+- lib/astrails/safe/local.rb
+- lib/astrails/safe/mysqldump.rb
+- lib/astrails/safe/pgdump.rb
+- lib/astrails/safe/pipe.rb
+- lib/astrails/safe/s3.rb
+- lib/astrails/safe/sink.rb
+- lib/astrails/safe/source.rb
+- lib/astrails/safe/stream.rb
+- lib/astrails/safe/tmp_file.rb
+- lib/astrails/safe.rb
+- lib/extensions
+- lib/extensions/mktmpdir.rb
+- templates/script.rb
+- Rakefile
+- LICENSE
+has_rdoc: true
+homepage: http://github.com/astrails/safe
+post_install_message: 
+rdoc_options: 
+- --inline-source
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: Backup filesystem and database (MySQL and PostgreSQL) to Amazon S3 (with encryption)
+test_files: []
+