siba-destination-aws-s3 0.1.0

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+*.swp

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in siba-destination-aws-s3.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,9 @@
+# A sample Guardfile           
+# More info at https://github.com/guard/guard#readme
+
+guard 'minitest', :notify=>false do
+  watch(%r|^test/unit/(.*\/)*test_(.*)\.rb|)
+  watch(%r|^lib/siba-destination-aws-s3/(.*\/)*([^/]+)\.rb|) do |m| 
+    "test/unit/#{m[1]}test_#{m[2]}.rb" unless m[2][0] == '.' 
+  end
+end

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2012 Evgeny Neumerzhitskiy
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,36 @@
+# Amazon S3 destination plugin for SIBA
+
+This plugin lets you keep your SIBA backups in Amazon S3 cloud storage.
+
+## Installation
+
+        $ gem install siba-destination-aws-s3
+
+## Usage
+
+Run `siba generate FILE` command to generate options file and select aws-s3 as destination plugin. After that edit the options file and set your Amazon S3 options.
+
+## License
+
+Copyright (c) 2011 Evgeny Neumerzhitskiy
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+

data/Rakefile

@@ -0,0 +1,28 @@
+require "bundler/gem_tasks"
+require 'rake/testtask'        
+
+namespace "test" do
+  desc "Run all unit tests"    
+  Rake::TestTask.new("unit") do |t|     
+    t.pattern = "test/unit/**/test*.rb"
+    t.libs << 'test'           
+  end
+
+  desc "Run all integration tests"
+  Rake::TestTask.new("integration") do |t|     
+    t.pattern = "test/integration/**/i9n_*.rb"
+    t.libs << 'test'           
+  end
+
+  desc "Run all integration tests"
+  task :i9n => ["test:integration"] do 
+  end
+end
+
+desc "Run all unit tests"
+task :test => ["test:unit"] do
+end
+
+desc "Run tests"               
+task :default => "test:unit"
+

data/lib/siba-destination-aws-s3.rb

@@ -0,0 +1,11 @@
+# encoding: UTF-8
+
+require "siba-destination-aws-s3/version"
+require "siba-destination-aws-s3/init"
+
+module Siba
+  module Destination
+    module AwsS3
+    end
+  end
+end

data/lib/siba-destination-aws-s3/cloud.rb

@@ -0,0 +1,139 @@
+# encoding: UTF-8
+
+require 'aws/s3'
+
+module Siba::Destination
+  module AwsS3
+    class Cloud
+      include Siba::FilePlug
+      include Siba::LoggerPlug
+
+      attr_accessor :bucket, :sub_dir, :access_key_id, :secret_key
+
+      def initialize(bucket, access_key_id, secret_key)
+        splitted_name = bucket.split("/")
+        @bucket = splitted_name.shift
+        @sub_dir = splitted_name.join("/")
+        @access_key_id = access_key_id
+        @secret_key = secret_key
+
+        check_connection
+      end
+
+      def upload(src_file)        
+        file_name = File.basename src_file
+        logger.info "Uploading backup file to Amazon S3: #{file_name}, bucket: #{bucket_with_path}"
+        access_and_close do
+          unless siba_file.file_file? src_file
+            raise Siba::Error, "Can not find backup file for uploading: #{src_file}"
+          end
+
+          File.open(src_file, "rb") do |file|
+            AWS::S3::S3Object.store path(file_name), file
+          end
+        end
+      end
+
+      def get_backups_list(backup_prefix)
+        backups = find_objects backup_prefix    
+        siba_file.run_this do
+          backups.map do |obj|
+            [File.basename(obj.key), obj.last_modified]
+          end
+        end
+      end
+
+      def restore_backup_to_dir(backup_name, dir)
+        access_and_close do
+          path_to_file = File.join dir, backup_name
+          open(path_to_file, 'wb') do |file|
+            file.write(get_file backup_name)
+          end
+        end
+      end
+
+      def exists?(file_name)
+        access_and_close do
+          AWS::S3::S3Object.exists? path(file_name)
+        end
+      end
+
+      def bucket_exists?
+        access_and_close do
+          AWS::S3::Service.buckets.any?{|a| a.name == bucket}
+        end
+      end
+
+      def find_objects(object_prefix)
+        access_and_close do
+          AWS::S3::Bucket.objects(prefix: path(object_prefix))
+        end
+      end
+
+      def get_file(file_name)
+        access_and_close do
+          begin
+            AWS::S3::S3Object.value path(file_name)
+          rescue
+            logger.error "Failed to get file #{file_name} from Amazon S3"
+            raise
+          end
+        end
+      end
+
+      def delete(file_name)
+        access_and_close do
+          begin
+            AWS::S3::S3Object.delete path(file_name)
+          rescue
+            logger.error "Failed to delete file #{file_name} from Amazon S3"
+            raise
+          end
+        end
+      end
+
+      def check_connection
+        siba_file.run_this do
+          raise Siba::Error, "Bucket '#{bucket}' does not exist." unless bucket_exists?
+          exists? "some_file"
+          logger.debug "Access to Amazon S3 is verified"
+        end
+      rescue Exception
+        logger.error "Can not connect to Amazon S3. Bucket: #{bucket_with_path}"
+        raise
+      end
+
+      private
+
+      def path(file)
+        return file if sub_dir.empty?
+        sub_dir + "/" + file
+      end
+
+      def bucket_with_path
+        return bucket if sub_dir.empty?
+        bucket + "/" + sub_dir
+      end
+
+      def access_and_close
+        siba_file.run_this do
+          begin
+            AWS::S3::Base.establish_connection!(
+              :access_key_id => access_key_id,
+              :secret_access_key => secret_key,
+              :use_ssl => true);
+
+              AWS::S3::Bucket.set_current_bucket_to bucket
+              AWS::S3::S3Object.set_current_bucket_to bucket
+              yield
+          ensure
+            begin
+              AWS::S3::Base.disconnect!
+            rescue
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/siba-destination-aws-s3/init.rb

@@ -0,0 +1,48 @@
+# encoding: UTF-8
+
+require 'siba-destination-aws-s3/cloud'
+
+module Siba::Destination
+  module AwsS3                 
+    class Init                 
+      include Siba::LoggerPlug
+
+      DEFAULT_ACCESS_KEY_ID_ENV_NAME = "SIBA_AMAZON_ACCESS_KEY_ID"
+      DEFAULT_SECRET_KEY_ENV_NAME = "SIBA_AMAZON_SECRET_ACCESS_KEY"
+
+      attr_accessor :cloud
+
+      def initialize(options)  
+        bucket = Siba::SibaCheck.options_string options, "bucket"
+
+        access_key_id_name = "access_key_id"
+        secret_key_name = "secret_access_key"
+        access_key_id = Siba::SibaCheck.options_string options, access_key_id_name, true, ENV[DEFAULT_ACCESS_KEY_ID_ENV_NAME]
+        secret_key = Siba::SibaCheck.options_string options, secret_key_name, true, ENV[DEFAULT_SECRET_KEY_ENV_NAME]
+
+        raise Siba::CheckError, "Missing '#{access_key_id_name}' option" if access_key_id.nil?
+        raise Siba::CheckError, "Missing '#{secret_key_name}' option" if secret_key.nil?
+
+        logger.info "Checking connection to Amazon S3"
+        @cloud = Siba::Destination::AwsS3::Cloud.new bucket, access_key_id, secret_key
+      end                      
+
+      def backup(path_to_backup_file) 
+        @cloud.upload path_to_backup_file
+      end
+
+      # Returns an array of two-element arrays:
+      # [backup_file_name, modification_time]
+      def get_backups_list(backup_name)
+        logger.info "Getting the list of backups from Amazon S3"
+        @cloud.get_backups_list backup_name
+      end
+
+      # Put backup file into dir
+      def restore(backup_name, dir)
+        logger.info "Downloading backup from Amazon S3"
+        @cloud.restore_backup_to_dir backup_name, dir
+      end
+    end
+  end
+end

data/lib/siba-destination-aws-s3/options.yml

@@ -0,0 +1,3 @@
+bucket: your-bucket/sub-dir
+access_key_id: your AWS S3 'access key id'
+secret_access_key: your AWS S3 'secret access key'

data/lib/siba-destination-aws-s3/version.rb

@@ -0,0 +1,9 @@
+# encoding: UTF-8
+
+module Siba
+  module Destination
+    module AwsS3
+      VERSION = "0.1.0"
+    end
+  end
+end

data/siba-destination-aws-s3.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "siba-destination-aws-s3/version"
+
+Gem::Specification.new do |s|
+  s.name        = "siba-destination-aws-s3"
+  s.version     = Siba::Destination::AwsS3::VERSION
+  s.authors     = ["Evgeny Neumerzhitskiy"]
+  s.email       = ["sausageskin@gmail.com"]
+  s.homepage    = ""
+  s.summary     = %q{Writes SIBA backup to Amazon S3 storage}
+  s.description = %q{An extension for SIBA gem. Allows to upload backups to Amazon S3 cloud storage.}
+
+  s.rubyforge_project = "siba-destination-aws-s3"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  
+  s.add_runtime_dependency     'siba', '~>0.4'
+  s.add_runtime_dependency     'aws-s3', '~>0.6'
+
+  s.add_development_dependency  'minitest', '~>2.10'
+  s.add_development_dependency  'rake', '~>0.9'
+  s.add_development_dependency  'guard-minitest', '~>0.4'
+end

data/test/helper/require.rb

@@ -0,0 +1,13 @@
+# encoding: UTF-8
+
+require 'rubygems'
+gem 'minitest' # loads minitest from gem, not the included one
+require 'minitest/autorun'     
+
+require 'siba'
+require 'siba/helpers/test/file_mock'
+require 'siba/helpers/test/kernel_mock'
+require 'siba/helpers/test/extend_test' 
+require 'siba/helpers/test/removable_constants'
+require 'siba/helpers/test/helper'
+

data/test/helper/require_integration.rb

@@ -0,0 +1,5 @@
+# encoding: UTF-8
+
+require 'helper/require'
+SibaTest.init_integration      
+

data/test/helper/require_unit.rb

@@ -0,0 +1,4 @@
+# encoding: UTF-8
+
+require 'helper/require'       
+SibaTest.init_unit

data/test/integration/i9n_cloud.rb

@@ -0,0 +1,58 @@
+# encoding: UTF-8
+
+require 'helper/require_integration'
+require 'siba-destination-aws-s3/init'
+
+describe Siba::Destination::AwsS3::Cloud do
+  before do
+    @cls = Siba::Destination::AwsS3::Cloud
+    @init = Siba::Destination::AwsS3::Init
+
+    bucket_env = "SIBA_TEST_AWS_S3_BUCKET_NAME"
+    access_key_id_env = @init::DEFAULT_ACCESS_KEY_ID_ENV_NAME
+    secret_key_env = @init::DEFAULT_SECRET_KEY_ENV_NAME
+
+    @bucket = ENV[bucket_env]
+    @access_key_id = ENV[access_key_id_env]
+    @secret_key = ENV[secret_key_env]
+
+    flunk "#{bucket_env} environment variable is not set" unless @bucket
+    flunk "#{access_key_id_env} environment variable is not set" unless @access_key_id
+    flunk "#{secret_key_env} environment variable is not set" unless @secret_key
+  end
+
+  it "should upload file to cloud and get the list" do
+    begin
+      @cloud = @cls.new @bucket, @access_key_id, @secret_key
+
+      backup_name = "awss3-u"
+      path_to_test_file = prepare_test_file backup_name
+      @cloud.upload path_to_test_file
+      
+      file_name = File.basename path_to_test_file
+      @cloud.exists?(file_name).must_equal true
+      @cloud.get_file(file_name).must_equal Siba::FileHelper.read(path_to_test_file)
+
+      # get list
+      list = @cloud.get_backups_list backup_name
+      full_backup_name = list[0][0]
+      full_backup_name.must_match /^#{backup_name}/
+      list[0][1].must_be_instance_of Time
+
+      # restore backup
+      dest_dir = mkdir_in_tmp_dir "awsbk"
+      @cloud.restore_backup_to_dir full_backup_name, dest_dir
+      path_to_dest_file = File.join dest_dir, full_backup_name
+      File.file?(path_to_dest_file).must_equal true
+      FileUtils.compare_file(path_to_test_file, path_to_dest_file).must_equal true
+    ensure
+      @cloud.delete(file_name)
+      @cloud.exists?(file_name).must_equal false
+    end
+  end
+
+  it "should find objects" do
+    @cloud = @cls.new @bucket, @access_key_id, @secret_key
+    @cloud.find_objects("my").must_be_instance_of Array
+  end
+end

data/test/unit/test_cloud.rb

@@ -0,0 +1,24 @@
+# encoding: UTF-8
+
+require 'helper/require_unit'
+require 'siba-destination-aws-s3/init'
+
+describe Siba::Destination::AwsS3::Cloud do
+  before do                    
+    @obj = Siba::Destination::AwsS3::Cloud.new "bucket", "two", "umi"
+  end
+
+  it "should assign variables" do
+    @obj.bucket.must_equal "bucket"
+    @obj.access_key_id.must_equal "two"
+    @obj.secret_key.must_equal "umi"
+  end
+
+  it "should upload" do
+    @obj.upload "/file"
+  end
+  
+  it "should upload" do
+    @obj.get_backups_list "backups"
+  end
+end

data/test/unit/test_init.rb

@@ -0,0 +1,54 @@
+# encoding: UTF-8
+
+require 'helper/require_unit'
+require 'siba-destination-aws-s3/init'
+
+describe Siba::Destination::AwsS3::Init do
+  before do                    
+    @obj = Siba::Destination::AwsS3::Init
+    @yml_path = File.expand_path('../yml', __FILE__)
+    @options_hash = load_options "valid" 
+    @plugin = @obj.new @options_hash
+  end
+
+  it "should load plugin" do
+    @plugin.must_be_instance_of Siba::Destination::AwsS3::Init
+    @plugin.cloud.must_be_instance_of Siba::Destination::AwsS3::Cloud
+  end
+
+  it "should not fail if keys are missing but env variables are defined" do
+    ENV[@obj::DEFAULT_ACCESS_KEY_ID_ENV_NAME] = "akd"
+    ENV[@obj::DEFAULT_SECRET_KEY_ENV_NAME] = "sk"
+    cloud = @obj.new({"bucket"=>"bucket"})
+    cloud.cloud.access_key_id.must_equal "akd"
+    cloud.cloud.secret_key.must_equal "sk"
+  end
+
+  it "should fail if keys are missing" do
+    ENV[@obj::DEFAULT_ACCESS_KEY_ID_ENV_NAME] = nil
+    ENV[@obj::DEFAULT_SECRET_KEY_ENV_NAME] = "sk"
+    ->{@obj.new({"bucket"=>"bucket"})}.must_raise Siba::CheckError
+
+    ENV[@obj::DEFAULT_ACCESS_KEY_ID_ENV_NAME] = "akd"
+    ENV[@obj::DEFAULT_SECRET_KEY_ENV_NAME] = nil
+    ->{@obj.new({"bucket"=>"bucket"})}.must_raise Siba::CheckError
+  end
+  
+  it "should fail when bucket is missing" do
+    ->{@obj.new({})}.must_raise Siba::CheckError
+    ->{@obj.new({"secret_key"=>"value"})}.must_raise Siba::CheckError
+    ->{@obj.new({"access_key_id"=>"value"})}.must_raise Siba::CheckError
+  end
+
+  it "should run backup" do
+    @plugin.backup "/file"
+  end
+  
+  it "should run get_backups_list" do
+    @plugin.get_backups_list "/file"
+  end
+
+  it "should run restore" do
+    @plugin.restore "backup_name", "/dir"
+  end
+end

data/test/unit/yml/valid.yml

@@ -0,0 +1,5 @@
+type: aws-s3
+bucket: bucket/subdir
+access_key_id: acces key id
+secret_access_key: secret key
+

metadata

@@ -0,0 +1,120 @@
+--- !ruby/object:Gem::Specification
+name: siba-destination-aws-s3
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Evgeny Neumerzhitskiy
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-01-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: siba
+  requirement: &85710210 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: *85710210
+- !ruby/object:Gem::Dependency
+  name: aws-s3
+  requirement: &85709390 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: *85709390
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: &85705900 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.10'
+  type: :development
+  prerelease: false
+  version_requirements: *85705900
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &85705530 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: *85705530
+- !ruby/object:Gem::Dependency
+  name: guard-minitest
+  requirement: &85705130 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :development
+  prerelease: false
+  version_requirements: *85705130
+description: An extension for SIBA gem. Allows to upload backups to Amazon S3 cloud
+  storage.
+email:
+- sausageskin@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Guardfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/siba-destination-aws-s3.rb
+- lib/siba-destination-aws-s3/cloud.rb
+- lib/siba-destination-aws-s3/init.rb
+- lib/siba-destination-aws-s3/options.yml
+- lib/siba-destination-aws-s3/version.rb
+- siba-destination-aws-s3.gemspec
+- test/helper/require.rb
+- test/helper/require_integration.rb
+- test/helper/require_unit.rb
+- test/integration/i9n_cloud.rb
+- test/unit/test_cloud.rb
+- test/unit/test_init.rb
+- test/unit/yml/valid.yml
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: siba-destination-aws-s3
+rubygems_version: 1.8.11
+signing_key: 
+specification_version: 3
+summary: Writes SIBA backup to Amazon S3 storage
+test_files: []