astrails-safe 0.3.0 → 0.3.1

data/CHANGELOG

@@ -1,3 +1,13 @@
+0.3.1
+
+* plain ftp support from seroy
+* mongodump support from Matt Berther
+
+0.3.0
+
+* switch to bundler
+* fixed the rspec
+
 0.2.8
 
 * ruby 1.9.2 compatibility (tests mostly)

data/README.markdown

@@ -4,8 +4,9 @@ Simple database and filesystem backups with S3 and Rackspace Cloud Files support
 
 * Home: [http://astrails.com/opensource/astrails-safe](http://astrails.com/opensource/astrails-safe)
 * Code: [http://github.com/astrails/safe](http://github.com/astrails/safe)
-* Blog: [http://blog.astrails.com/astrails-safe](http://blog.astrails.com/astrails-safe)
+* Blog: [http://astrails.com/blog/astrails-safe](http://astrails.com/blog/astrails-safe)
 
+[![Build Status](https://travis-ci.org/astrails/safe.png)](https://travis-ci.org/astrails/safe)
 [![Code Climate](https://codeclimate.com/github/astrails/safe.png)](https://codeclimate.com/github/astrails/safe)
 
 ## Motivation
@@ -34,6 +35,8 @@ The following functionality was contributed by astrails-safe users:
 * improved config file parsing (by Fedor Kocherga <fkocherga@gmail.com>)
 * mysql password file quoting (by Jonathan Sutherland <jonathan.sutherland@gmail.com>)
 * Rackspace Cloud Files support (by H. Wade Minter <minter@lunenburg.org>)
+* Plan FTP support (by seroy <seroy@bk.ru>)
+* mongodump support (by Matt Berther <matt@mattberther.com>)
 
 Thanks to all :)
 
@@ -234,4 +237,4 @@ The procedure to create and transfer the key is as follows:
 
 ## Copyright
 
-Copyright (c) 2010 Astrails Ltd. See LICENSE.txt for details.
+Copyright (c) 2010-2013 Astrails Ltd. See LICENSE.txt for details.

data/TODO

@@ -1,3 +1,22 @@
+- refactor
+  - refactor out global variables. pass a config object around instead
+  - common logging
+  - remove 1.8.6 support
+  - module registry
+  - base => prefix ?
+  - move requires into specific modules
+
+- features
+  - remote-only s3 support
+  - generic notifier support
+    - email notifier
+    - hipchat
+  - generic error notifier support
+    - email
+    - hipchat
+
+
+
 - add 'silent'
 - handle errors from mysqldump
 - check that gpg is installed

data/astrails-safe.gemspec

@@ -11,7 +11,7 @@ Gem::Specification.new do |spec|
   spec.description = <<-DESC
 Astrails-Safe is a simple tool to backup databases (MySQL and PostgreSQL), Subversion repositories (with svndump) and just files.
 Backups can be stored locally or remotely and can be enctypted.
-Remote storage is supported on Amazon S3, Rackspace Cloud Files, or just plain SFTP.
+Remote storage is supported on Amazon S3, Rackspace Cloud Files, or just plain FTP/SFTP.
 DESC
   spec.summary       = %Q{Backup filesystem and databases (MySQL and PostgreSQL) locally or to a remote server/service (with encryption)}
   spec.homepage      = "http://astrails.com/astrails-safe"

data/bin/astrails-safe

@@ -2,10 +2,8 @@
 
 require 'rubygems'
 
-# require 'ruby-debug'
-$:.unshift File.expand_path("../../lib", __FILE__)
-
 require 'astrails/safe'
+
 include Astrails::Safe
 
 def die(msg)

data/lib/astrails/safe.rb

@@ -3,6 +3,7 @@ require "astrails/safe/version"
 require "aws/s3"
 require "cloudfiles"
 require 'net/sftp'
+require 'net/ftp'
 require 'fileutils'
 require 'benchmark'
 
@@ -23,6 +24,7 @@ require 'astrails/safe/mysqldump'
 require 'astrails/safe/pgdump'
 require 'astrails/safe/archive'
 require 'astrails/safe/svndump'
+require 'astrails/safe/mongodump'
 
 require 'astrails/safe/pipe'
 require 'astrails/safe/gpg'
@@ -33,6 +35,7 @@ require 'astrails/safe/local'
 require 'astrails/safe/s3'
 require 'astrails/safe/cloudfiles'
 require 'astrails/safe/sftp'
+require 'astrails/safe/ftp'
 
 module Astrails
   module Safe
@@ -45,12 +48,13 @@ module Astrails
 
       [[Mysqldump, [:mysqldump, :databases]],
        [Pgdump,    [:pgdump,    :databases]],
+       [Mongodump, [:mongodump, :databases]],
        [Archive,   [:tar,       :archives]],
        [Svndump,   [:svndump,   :repos]]
       ].each do |klass, path|
         if collection = config[*path]
           collection.each do |name, c|
-            klass.new(name, c).backup.run(c, :gpg, :gzip, :local, :s3, :cloudfiles, :sftp)
+            klass.new(name, c).backup.run(c, :gpg, :gzip, :local, :s3, :cloudfiles, :sftp, :ftp)
           end
         end
       end

data/lib/astrails/safe/config/builder.rb

@@ -4,7 +4,7 @@ module Astrails
       class Builder
         COLLECTIONS = %w/database archive repo/
         ITEMS = %w/s3 cloudfiles key secret bucket api_key container service_net path gpg password keep local mysqldump pgdump command options
-        user host port socket skip_tables tar files exclude filename svndump repo_path sftp/
+        user host port socket skip_tables tar files exclude filename svndump repo_path sftp ftp mongodump/
         NAMES = COLLECTIONS + ITEMS
         def initialize(node)
           @node = node

data/lib/astrails/safe/ftp.rb

@@ -0,0 +1,85 @@
+module Astrails
+  module Safe
+    class Ftp < Sink
+
+      protected
+
+      def active?
+        host && user
+      end
+
+      def path
+        @path ||= expand(config[:ftp, :path] || config[:local, :path] || ":kind/:id")
+      end
+
+      def save
+        raise RuntimeError, "pipe-streaming not supported for FTP." unless @backup.path
+
+        puts "Uploading #{host}:#{full_path} via FTP" if $_VERBOSE || $DRY_RUN
+
+        unless $DRY_RUN || $LOCAL
+          if !port
+            port = 21
+          end
+          Net::FTP.open(host) do |ftp|
+            ftp.connect(host, port)
+            ftp.login(user, password)
+            puts "Sending #{@backup.path} to #{full_path}" if $_VERBOSE
+            begin
+              ftp.put(@backup.path, full_path)
+            rescue Net::FTPPermError
+              puts "Ensuring remote path (#{path}) exists" if $_VERBOSE
+            end
+          end
+          puts "...done" if $_VERBOSE
+        end
+      end
+
+      def cleanup
+        return if $LOCAL || $DRY_RUN
+
+        return unless keep = @config[:keep, :ftp]
+
+        puts "listing files: #{host}:#{base}*" if $_VERBOSE
+        if !port
+           port = 21
+        end
+        Net::FTP.open(host) do |ftp|
+          ftp.connect(host, port)
+          ftp.login(user, password)
+          files = ftp.nlst(path)
+          pattern = File.basename("#{base}")
+          files = files.reject{ |x| !x.start_with?(pattern)}
+          puts files.collect {|x| x} if $_VERBOSE
+
+          files = files.
+            collect {|x| x }.
+            sort
+
+          cleanup_with_limit(files, keep) do |f|
+            file = File.join(path, f)
+            puts "removing ftp file #{host}:#{file}" if $DRY_RUN || $_VERBOSE
+            ftp.delete(file) unless $DRY_RUN || $LOCAL
+          end
+        end
+      end
+
+      def host
+        @config[:ftp, :host]
+      end
+
+      def user
+        @config[:ftp, :user]
+      end
+
+      def password
+        @config[:ftp, :password]
+      end
+
+      def port
+        @config[:ftp, :port]
+      end
+
+    end
+  end
+end

data/lib/astrails/safe/mongodump.rb

@@ -0,0 +1,23 @@
+module Astrails
+  module Safe
+    class Mongodump < Source
+      
+      def command
+        opts = []
+        opts << "--host #{@config[:host]}" if @config[:host]
+        opts << "-u #{@config[:user]}" if @config[:user]
+        opts << "-p #{@config[:password]}" if @config[:password]
+        opts << "--out #{output_directory}"
+      
+        "mongodump -q \"{xxxx : { \\$ne : 0 } }\" --db #{@id} #{opts.join(" ")} && cd #{output_directory} && tar cf - ."
+      end
+    
+      def extension; '.tar'; end
+      
+      protected
+      def output_directory
+        File.join(TmpFile.tmproot, "mongodump")
+      end
+    end
+  end
+end

data/lib/astrails/safe/version.rb

@@ -1,5 +1,5 @@
 module Astrails
   module Safe
-    VERSION = "0.3.0"
+    VERSION = "0.3.1"
   end
 end

data/spec/integration/cleanup_spec.rb

@@ -55,7 +55,7 @@ describe "tar backup" do
   end
 
   it "should NOT remove backups with base having same prefix" do
-    Dir["#{@dst}/archive/archive-foobar.*"].should == ["#{@dst}/archive/archive-foobar.000001.tar.gz", "#{@dst}/archive/archive-foobar.000002.tar.gz"]
+    Dir["#{@dst}/archive/archive-foobar.*"].sort.should == ["#{@dst}/archive/archive-foobar.000001.tar.gz", "#{@dst}/archive/archive-foobar.000002.tar.gz"]
   end
 
 end

data/spec/unit/config_spec.rb

@@ -101,6 +101,12 @@ describe Astrails::Safe::Config do
         end
       end
 
+      mongodump do
+        host "host"
+        database "database"
+        user "user"
+        password "password"
+      end
     end
 
     expected = {
@@ -177,6 +183,15 @@ describe Astrails::Safe::Config do
           "misc" => { "files" => ["/backup/*.rb"] },
         },
       },
+      
+      "mongodump" => {
+        "host" => "host",
+        "databases" => {
+          "database" => {}
+        },
+        "user" => "user",
+        "password" => "password"
+      }      
     }
 
     config.to_hash.should == expected

data/spec/unit/mongodump_spec.rb

@@ -0,0 +1,54 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe Astrails::Safe::Mongodump do
+  def def_config
+    {
+      :host => 'prod.example.com',
+      :user => 'testuser',
+      :password => 'p4ssw0rd',
+    }
+  end
+  
+  def mongodump(id = :foo, config = def_config)
+    Astrails::Safe::Mongodump.new(id, Astrails::Safe::Config::Node.new(nil, config))
+  end
+  
+  before(:each) do
+    stub(Time).now.stub!.strftime {"NOW"}
+    @output_folder = File.join(Astrails::Safe::TmpFile.tmproot, 'mongodump')
+  end
+  
+  after(:each) { Astrails::Safe::TmpFile.cleanup }
+  
+  describe :backup do
+    before(:each) do
+      @mongo = mongodump
+    end
+    
+    {
+      :id => "foo",
+      :kind => "mongodump",
+      :extension => ".tar",
+      :filename => "mongodump-foo.NOW"
+    }.each do |k, v|
+      it "should set #{k} to #{v}" do
+        @mongo.backup.send(k).should == v
+      end
+    end
+    
+    it "should set the command" do
+      @mongo.backup.send(:command).should == "mongodump -q \"{xxxx : { \\$ne : 0 } }\" --db foo --host prod.example.com -u testuser -p p4ssw0rd --out #{@output_folder} && cd #{@output_folder} && tar cf - ."
+    end
+    
+    {
+      :host => "--host ",
+      :user => "-u ",
+      :password => "-p "
+    }.each do |key, v|    
+      it "should not add #{key} to command if it is not present" do
+        @mongo = mongodump(:foo, def_config.reject! {|k,v| k == key})
+        @mongo.backup.send(:command).should_not =~ /#{v}/
+      end
+    end
+  end
+end

data/templates/script.rb

@@ -46,6 +46,15 @@ safe do
   #   path ":kind/:id" # this is the default
   # end
 
+  ## uncomment to enable uploads via FTP
+  # ftp do
+  #   host "YOUR_REMOTE_HOSTNAME"
+  #   user "YOUR_REMOTE_USERNAME"
+  #   # port "NON STANDARD FTP PORT"
+  #   password "YOUR_REMOTE_PASSWORD"
+  #   path ":kind/:id" # this is the default
+  # end
+
   ## uncomment to enable GPG encryption.
   ## Note: you can use public 'key' or symmetric password but not both!
   # gpg do

metadata

@@ -2,14 +2,14 @@
 name: astrails-safe
 version: !ruby/object:Gem::Version
   prerelease: 
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Vitaly Kushner
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-06-01 00:00:00.000000000 Z
+date: 2013-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-s3
@@ -128,7 +128,7 @@ description: ! 'Astrails-Safe is a simple tool to backup databases (MySQL and Po
 
   Backups can be stored locally or remotely and can be enctypted.
 
-  Remote storage is supported on Amazon S3, Rackspace Cloud Files, or just plain SFTP.
+  Remote storage is supported on Amazon S3, Rackspace Cloud Files, or just plain FTP/SFTP.
 
 '
 email:
@@ -155,9 +155,11 @@ files:
 - lib/astrails/safe/cloudfiles.rb
 - lib/astrails/safe/config/builder.rb
 - lib/astrails/safe/config/node.rb
+- lib/astrails/safe/ftp.rb
 - lib/astrails/safe/gpg.rb
 - lib/astrails/safe/gzip.rb
 - lib/astrails/safe/local.rb
+- lib/astrails/safe/mongodump.rb
 - lib/astrails/safe/mysqldump.rb
 - lib/astrails/safe/pgdump.rb
 - lib/astrails/safe/pipe.rb
@@ -179,6 +181,7 @@ files:
 - spec/unit/gpg_spec.rb
 - spec/unit/gzip_spec.rb
 - spec/unit/local_spec.rb
+- spec/unit/mongodump_spec.rb
 - spec/unit/mysqldump_spec.rb
 - spec/unit/pgdump_spec.rb
 - spec/unit/s3_spec.rb
@@ -197,7 +200,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       segments:
       - 0
-      hash: 1115506129816519000
+      hash: -3388246215765572723
       version: '0'
   none: false
 required_rubygems_version: !ruby/object:Gem::Requirement
@@ -206,7 +209,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       segments:
       - 0
-      hash: 1115506129816519000
+      hash: -3388246215765572723
       version: '0'
   none: false
 requirements: []
@@ -226,6 +229,7 @@ test_files:
 - spec/unit/gpg_spec.rb
 - spec/unit/gzip_spec.rb
 - spec/unit/local_spec.rb
+- spec/unit/mongodump_spec.rb
 - spec/unit/mysqldump_spec.rb
 - spec/unit/pgdump_spec.rb
 - spec/unit/s3_spec.rb