astrails-safe 0.2.0 → 0.2.1

data/README.markdown

@@ -20,12 +20,23 @@ We needed a backup solution that will satisfy the following requirements:
 
 And since we didn't find any, we wrote our own :)
 
-Note
-----
+Contributions
+-------------
 
-Support for pg_dump and svndump was contributed but since I don't personally use them i don't have an easy
-way of testing it. So if you use Subversion or PostgreSQL please test the new functionality and report if
-there are any problems. Issues tracker is [here](http://github.com/astrails/safe/issues)
+The following functionality was contributed by astrails-safe users:
+
+* PostgreSQL dump using pg_dump (by Mark Mansour <mark@stateofflux.com>)
+* Subversion dump using svndump (by Richard Luther <richard.luther@gmail.com>)
+* SFTP remote storage (by Adam <adam@mediadrive.ca>)
+* benchmarking output (By Neer)
+
+
+Thanks to all :)
+
+Reporting problems
+------------------
+
+Please report problems at the [Issues tracker](http://github.com/astrails/safe/issues)
 
 Usage
 -----
@@ -118,6 +129,12 @@ Example configuration
       path "servers/alpha/:kind/:id"
     end
 
+    sftp do
+      host "sftp.astrails.com"
+      user "astrails"
+      password "ssh password for sftp"
+    end
+
     gpg do
       # symmetric encryption key
       # password "qwe"

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
 :major: 0
 :minor: 2
-:patch: 0
+:patch: 1

data/examples/integration/cleanup_example.rb

@@ -0,0 +1,62 @@
+require File.expand_path(File.dirname(__FILE__) + '/../example_helper')
+
+require "fileutils"
+include FileUtils
+
+describe "tar backup" do
+  before(:all) do
+    # need both local and instance vars
+    # instance variables are used in tests
+    # local variables are used in the backup definition (instance vars can't be seen)
+    @root = root = "tmp/cleanup_example"
+
+    # clean state
+    rm_rf @root
+    mkdir_p @root
+
+    # create source tree
+    @src = src = "#{@root}/src"
+    mkdir_p src
+
+    File.open(qwe = "#{@src}/qwe", "w") {|f| f.write("qwe") }
+
+    @dst = dst = "#{@root}/backup"
+    mkdir_p "#{@dst}/archive"
+
+    @now = Time.now
+    @timestamp = @now.strftime("%y%m%d-%H%M")
+
+    stub(Time).now {@now} # Freeze
+
+    cp qwe, "#{dst}/archive/archive-foo.000001.tar.gz"
+    cp qwe, "#{dst}/archive/archive-foo.000002.tar.gz"
+    cp qwe, "#{dst}/archive/archive-foobar.000001.tar.gz"
+    cp qwe, "#{dst}/archive/archive-foobar.000002.tar.gz"
+
+    Astrails::Safe.safe do
+      local :path => "#{dst}/:kind"
+      tar do
+        keep :local => 1 # only leave the latest
+        archive :foo do
+          files src
+        end
+      end
+    end
+
+    @backup = "#{dst}/archive/archive-foo.#{@timestamp}.tar.gz"
+  end
+
+  it "should create backup file" do
+    puts "Expecting: #{@backup}"
+    File.exists?(@backup).should be_true
+  end
+
+  it "should remove old backups" do
+    Dir["#{@dst}/archive/archive-foo.*"].should == [@backup]
+  end
+
+  it "should NOT remove backups with base having same prefix" do
+    Dir["#{@dst}/archive/archive-foobar.*"].should == ["#{@dst}/archive/archive-foobar.000001.tar.gz", "#{@dst}/archive/archive-foobar.000002.tar.gz"]
+  end
+
+end

data/examples/unit/config_example.rb

@@ -14,6 +14,12 @@ describe Astrails::Safe::Config do
         path "path1"
       end
 
+      sftp do
+        user "sftp user"
+        password "sftp password"
+        host "sftp host"
+      end
+
       gpg do
         key "gpg-key"
         password "astrails"
@@ -107,6 +113,12 @@ describe Astrails::Safe::Config do
         "path" => "path1",
       },
 
+      "sftp" => {
+        "user" => "sftp user",
+        "password" => "sftp password",
+        "host" => "sftp host",
+      },
+
       "gpg" => {"password" => "astrails", "key" => "gpg-key"},
 
       "keep" => {"s3" => 20, "local" => 4},

data/examples/unit/local_example.rb

@@ -82,8 +82,8 @@ describe Astrails::Safe::Local do
   describe :cleanup do
     before(:each) do
       @local = local
-      @files = [4,1,3,2].to_a.map { |i| "aaaaa#{i}" }
-      stub(Dir).[](anything) {@files}
+      @files = [4,1,3,2].to_a.map { |i| "/mysqldump~blog~NoW/qweqwe.#{i}" }
+      stub(Dir).[]("/mysqldump~blog~NoW/qweqwe.*") {@files}
       stub(File).file?(anything) {true}
       stub(File).size(anything) {1}
       stub(File).unlink
@@ -96,8 +96,8 @@ describe Astrails::Safe::Local do
     end
 
     it "should delete extra files" do
-      mock(File).unlink("aaaaa1")
-      mock(File).unlink("aaaaa2")
+      mock(File).unlink("/mysqldump~blog~NoW/qweqwe.1")
+      mock(File).unlink("/mysqldump~blog~NoW/qweqwe.2")
       @local.send :cleanup
     end
   end

data/examples/unit/s3_example.rb

@@ -39,7 +39,7 @@ describe Astrails::Safe::S3 do
 
       @files = [4,1,3,2].to_a.map { |i| stub(o = {}).key {"aaaaa#{i}"}; o }
 
-      stub(AWS::S3::Bucket).objects("_bucket", :prefix => "_kind/_id/_kind-_id", :max_keys => 4) {@files}
+      stub(AWS::S3::Bucket).objects("_bucket", :prefix => "_kind/_id/_kind-_id.", :max_keys => 4) {@files}
       stub(AWS::S3::Bucket).find("_bucket").stub![anything].stub!.delete
     end
 

data/lib/astrails/safe.rb

@@ -1,5 +1,7 @@
 require "aws/s3"
+require 'net/sftp'
 require 'fileutils'
+require 'benchmark'
 
 require 'tempfile'
 require 'extensions/mktmpdir'
@@ -28,7 +30,7 @@ require 'astrails/safe/gzip'
 require 'astrails/safe/sink'
 require 'astrails/safe/local'
 require 'astrails/safe/s3'
-
+require 'astrails/safe/sftp'
 
 module Astrails
   module Safe
@@ -46,7 +48,7 @@ module Astrails
       ].each do |klass, path|
         if collection = config[*path]
           collection.each do |name, config|
-            klass.new(name, config).backup.run(config, :gpg, :gzip, :local, :s3)
+            klass.new(name, config).backup.run(config, :gpg, :gzip, :local, :s3, :sftp)
           end
         end
       end

data/lib/astrails/safe/config/builder.rb

@@ -4,7 +4,7 @@ module Astrails
       class Builder
         COLLECTIONS = %w/database archive repo/
         ITEMS = %w/s3 key secret bucket path gpg password keep local mysqldump pgdump options
-        user host port socket skip_tables tar files exclude filename svndump repo_path/
+        user host port socket skip_tables tar files exclude filename svndump repo_path sftp/
         NAMES = COLLECTIONS + ITEMS
         def initialize(node)
           @node = node

data/lib/astrails/safe/local.rb

@@ -19,7 +19,10 @@ module Astrails
 
         unless $DRY_RUN
           FileUtils.mkdir_p(path) unless File.directory?(path)
-          system "#{@backup.command}>#{@backup.path = full_path}"
+          benchmark = Benchmark.realtime do
+            system "#{@backup.command}>#{@backup.path = full_path}"
+          end
+          puts("command took " + sprintf("%.2f", benchmark) + " second(s).") if $_VERBOSE
         end
 
       end

data/lib/astrails/safe/s3.rb

@@ -20,11 +20,14 @@ module Astrails
 
         puts "Uploading #{bucket}:#{full_path}" if $_VERBOSE || $DRY_RUN
         unless $DRY_RUN || $LOCAL
-          AWS::S3::Bucket.create(bucket)
-          File.open(@backup.path) do |file|
-            AWS::S3::S3Object.store(full_path, file, bucket)
+          benchmark = Benchmark.realtime do
+            AWS::S3::Bucket.create(bucket)
+            File.open(@backup.path) do |file|
+              AWS::S3::S3Object.store(full_path, file, bucket)
+            end
           end
           puts "...done" if $_VERBOSE
+          puts("Upload took " + sprintf("%.2f", benchmark) + " second(s).") if $_VERBOSE
         end
       end
 

data/lib/astrails/safe/sftp.rb

@@ -0,0 +1,77 @@
+module Astrails
+  module Safe
+    class Sftp < Sink
+
+      protected
+
+      def active?
+        host && user
+      end
+
+      def path
+        @path ||= expand(config[:sftp, :path] || config[:local, :path] || ":kind/:id")
+      end
+
+      def save
+        puts "Uploading #{host}:#{full_path} via SFTP" if $_VERBOSE || $DRY_RUN
+
+        unless $DRY_RUN || $LOCAL
+          opts = {}
+          opts[:password] = password if password
+          Net::SFTP.start(host, user, opts) do |sftp|
+            puts "Sending #{@backup.path} to #{full_path}" if $_VERBOSE
+            begin
+              sftp.upload! @backup.path, full_path
+            rescue Net::SFTP::StatusException
+              puts "Ensuring remote path (#{path}) exists" if $_VERBOSE
+              # mkdir -p
+              folders = path.split('/')
+              folders.each_index do |i|
+                folder = folders[0..i].join('/')
+                puts "Creating #{folder} on remote" if $_VERBOSE
+                sftp.mkdir!(folder) rescue Net::SFTP::StatusException
+              end
+              retry
+            end
+          end
+          puts "...done" if $_VERBOSE
+        end
+      end
+
+      def cleanup
+        return if $LOCAL || $DRY_RUN
+
+        return unless keep = @config[:keep, :sftp]
+
+        puts "listing files in #{host}:#{path}" if $_VERBOSE
+        Net::SFTP.start(host, user, :password => password) do |sftp|
+          files = sftp.dir.glob(path, '*')
+
+          puts files.collect {|x| x.name } if $_VERBOSE
+
+          files = files.
+            collect {|x| x.name }.
+            sort
+
+          cleanup_with_limit(files, keep) do |f|
+            file = File.join(path, f)
+            puts "removing sftp file #{host}:#{file}" if $DRY_RUN || $_VERBOSE
+            sftp.remove!(file) unless $DRY_RUN || $LOCAL
+          end
+        end
+      end
+
+      def host
+        @config[:sftp, :host]
+      end
+
+      def user
+        @config[:sftp, :user]
+      end
+
+      def password
+        @config[:sftp, :password]
+      end
+    end
+  end
+end

data/lib/astrails/safe/sink.rb

@@ -13,7 +13,7 @@ module Astrails
 
       # path is defined in subclass
       def base
-        @base ||= File.join(path, File.basename(@backup.filename).split(".").first)
+        @base ||= File.join(path, File.basename(@backup.filename).split(".").first + '.')
       end
 
       def full_path

data/templates/script.rb

@@ -24,6 +24,14 @@ safe do
   ## alternative style:
   # s3 :key => YOUR_S3_KEY, :secret => YOUR_S3_SECRET, :bucket => S3_BUCKET
 
+  ## uncomment to enable uploads via SFTP
+  # sftp do
+  #   host "YOUR_REMOTE_HOSTNAME"
+  #   user "YOUR_REMOTE_USERNAME"
+  #   password "YOUR_REMOTE_PASSWORD"
+  #   path ":kind/:id" # this is the default
+  # end
+
   ## uncomment to enable GPG encryption.
   ## Note: you can use public 'key' or symmetric password but not both!
   # gpg do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: astrails-safe
 version: !ruby/object:Gem::Version 
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors: 
 - Astrails Ltd.
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-06-04 00:00:00 -07:00
+date: 2009-07-05 00:00:00 -07:00
 default_executable: astrails-safe
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -40,6 +40,7 @@ files:
 - bin/astrails-safe
 - examples/example_helper.rb
 - examples/integration/archive_integration_example.rb
+- examples/integration/cleanup_example.rb
 - examples/unit/archive_example.rb
 - examples/unit/config_example.rb
 - examples/unit/gpg_example.rb
@@ -61,6 +62,7 @@ files:
 - lib/astrails/safe/pgdump.rb
 - lib/astrails/safe/pipe.rb
 - lib/astrails/safe/s3.rb
+- lib/astrails/safe/sftp.rb
 - lib/astrails/safe/sink.rb
 - lib/astrails/safe/source.rb
 - lib/astrails/safe/stream.rb
@@ -95,14 +97,15 @@ signing_key:
 specification_version: 2
 summary: Backup filesystem and databases (MySQL and PostgreSQL) to Amazon S3 (with encryption)
 test_files: 
-- examples/integration/archive_integration_example.rb
 - examples/example_helper.rb
-- examples/unit/gzip_example.rb
-- examples/unit/s3_example.rb
-- examples/unit/mysqldump_example.rb
-- examples/unit/gpg_example.rb
+- examples/integration/archive_integration_example.rb
+- examples/integration/cleanup_example.rb
 - examples/unit/archive_example.rb
-- examples/unit/svndump_example.rb
 - examples/unit/config_example.rb
+- examples/unit/gpg_example.rb
+- examples/unit/gzip_example.rb
 - examples/unit/local_example.rb
+- examples/unit/mysqldump_example.rb
 - examples/unit/pgdump_example.rb
+- examples/unit/s3_example.rb
+- examples/unit/svndump_example.rb