s3rsync 0.1.3

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MjAyMmU2ZDVkNmM1YTdlNDdjOWNhYWZjY2FhN2U4MzJiY2I2MzRmZg==
+  data.tar.gz: !binary |-
+    YTkzYzFjNWFkYWE1MTk3NzRjMmYxY2Y0ZjU5MWE4MTY1Yzk3NWNmNg==
+SHA512:
+  metadata.gz: !binary |-
+    N2UxNTc0ZGNmZTE5ODkyNzcwYzE1NzczMGY0ZDM0MjM4MjJmOWM0YzVhZmM4
+    YzIyM2Q5OWU0MjU3OTI0YWM0NGE1MGZlZGMzZDdlNmMxN2MzYjNmYWQ0ZjUz
+    OGFjOGZiYjJhOWU1Yzg3N2Q5Y2FiOWU1NjI2N2Y5MWE4MDBjMWE=
+  data.tar.gz: !binary |-
+    OTE3NjU4MWQ2YWY0ZTJlNDVjMWU2Nzk0ZTk1NGRkYTlkOWYyNzQ0ZjNhMjYw
+    ZDNjMjlhYTJlYzJkNjBjNzEzOGRmNTI5YTZkYTA1YzQyNmUwNTNhYzBkN2Mx
+    NDFkMDdhODBjYWVlNmY1MDgyN2ZkYmVhNTJlMmFjMDY5MWM1MDU=

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.travis.yml

@@ -0,0 +1,13 @@
+language: ruby
+rvm:
+ - 1.9.3
+ - 2.1.2
+deploy:
+  provider: rubygems
+  api_key:
+    secure: O4vHRY7yoWgWpcp+dTqFwVjGSjeYnGDFbRbvgfjxbSWxIGhtD3HbuUm62ncb83IIr7eCTqIQkYXGsndvsd5k3m4xJhJA8rkxPWri/B+ZdOqwSChov65Raf1I+gUW/ZAgfyMvFBlNB4jEPFe46eL/2N1AQtLDeq1ZdJaUwYC2dew=
+  gem: s3rsync
+  on:
+    all_branches: true
+    tags: true
+    repo: idevops/s3rsync

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in s3rsync.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,9 @@
+#notification :gntp
+notification :growl
+
+guard :rspec, cmd: 'bundle exec rspec' do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/s3rsync/(.+)\.rb$}) { |m| "spec/#{m[1]}_spec.rb" }
+  watch('lib/s3rsync.rb') { "spec" }
+  watch('spec/spec_helper.rb') { "spec" }
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Innovid Ltd.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,36 @@
+# S3rsync
+
+============================================
+[![Gem Version](http://img.shields.io/gem/v/s3rsync.svg)][gem]
+[![Build Status](http://img.shields.io/travis/idevops/s3rsync.svg)][travis]
+
+[gem]: https://rubygems.org/gems/s3rsync
+[travis]: http://travis-ci.org/idevops/s3rsync
+
+Wrapper gem to sync files with s3cmd tool
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 's3rsync'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install s3rsync
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it ( http://github.com/<my-github-username>/s3rsync/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+
+task :default => [:spec]
+desc 'run Rspec specs'
+task :spec do
+  sh 'rspec spec'
+end
+

data/bin/s3rsync

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require 's3rsync/cli'
+S3Rsync::CLI.start

data/lib/s3rsync/base.rb

@@ -0,0 +1,32 @@
+require 'logger'
+
+module S3Rsync
+  class Base
+    def init_logger
+      logger = Logger.new(STDOUT)
+      logger.level = (!ENV['DEBUG'].nil?) ? Logger::DEBUG : Logger::INFO
+      logger.debug "logger.level: #{logger.level}"
+      logger
+    end
+    
+    def error_exit(msg)
+      @log.error msg
+      return 1
+    end
+    
+    def pass_cli(cmd, valid_out_pattern = nil)
+      cmd += ' 2>&1'
+      @log.debug "running cmd: #{cmd}"
+      cmd_out = `#{cmd}`
+      success_exec = $?.success?
+      @log.info "cmd_out: #{cmd_out}"
+      @log.debug "success_exec: #{success_exec}"
+    
+      valid_content = valid_out_pattern.nil? ? true : !cmd_out.match(/#{valid_out_pattern}/).nil?
+      @log.debug "valid_content: #{valid_content}"
+    
+      error_exit "Not successful execution or not valid_content! out: #{cmd_out}" if (!success_exec || !valid_content)
+      cmd_out
+    end
+  end
+end

data/lib/s3rsync/cli.rb

@@ -0,0 +1,30 @@
+require 'thor'
+
+module S3Rsync
+  class CLI < Thor
+    DEFAULT_SETTINGS = {
+      :lock_file_path => "default",
+    }
+
+    desc "upload", "Upload"
+    method_option :dir, :aliases => "-d", :required => true
+    method_option :config_path, :aliases => "-c", :required => true
+    method_option :s3_prefix, :aliases => "-p", :default => 's3rsync'
+    def upload
+      require 's3rsync/sync'
+      sync = Sync.new(options)
+      sync.run(:upload)
+    end
+    
+    desc "download", "Download"
+    method_option :dir, :aliases => "-d", :required => true
+    method_option :config_path, :aliases => "-c", :required => true
+    method_option :s3_prefix, :aliases => "-p", :default => 's3rsync'
+    def download
+      require 's3rsync/sync'
+      sync = Sync.new(options)
+      sync.run(:download)
+    end
+
+  end
+end

data/lib/s3rsync/sync.rb

@@ -0,0 +1,53 @@
+require "s3rsync"
+require "json"
+
+module S3Rsync
+  class Sync < Base
+  
+    def initialize(opt)
+      @log = init_logger
+      @log.debug "opt: #{opt.inspect}"
+      @params = {}
+      @params[:dir]         = opt['dir']
+      @params[:s3_prefix]   = opt['s3_prefix']
+      @params[:config_path] = opt['config_path']
+      @s3cmd = 's3cmd'
+      
+      @log.debug "@params[:config_path]: #{@params[:config_path]}"
+      begin
+        conf = JSON.parse File.read(@params[:config_path])
+      rescue Exception => e
+        mgs = "ERROR: opening or parsing config file failed: #{e.inspect}"
+        @log.error mgs
+        raise mgs
+        exit 1
+      end
+    
+      @log.debug "conf: #{conf}"
+      @params[:s3_bucket] = conf['env.static.s3.bucket']
+      @params[:s3_access] = conf['env.static.s3.key.user']
+      @params[:s3_secret] = conf['env.static.s3.key.secret']
+      @log.debug "@params: #{@params.inspect}"
+    end
+  
+    def run(mode = :upload)
+      sync_locations = "#{@params[:dir]} s3://#{@params[:s3_bucket]}/#{@params[:s3_prefix]}/"
+      @log.debug "original sync_locations: #{sync_locations}"
+      if mode == :download
+        @log.debug "reversing sync_locations"
+        sync_locations = sync_locations.split(' ').reverse.join(' ')
+        @log.debug "download sync_locations: #{sync_locations}"
+      end
+      sync_output = pass_cli "s3cmd sync #{sync_locations}"
+      @log.debug "sync_output: #{sync_output}"
+      if (sync_output == 1) || sync_output.downcase.include?('error')
+        @log.error "errors found in output: #{sync_output}"
+        return 1
+      else
+        return 0
+      end
+    end
+    
+  end
+end
+

data/lib/s3rsync/version.rb

@@ -0,0 +1,3 @@
+module S3rsync
+  VERSION = "0.1.3"
+end

data/lib/s3rsync.rb

@@ -0,0 +1,7 @@
+require "s3rsync/version"
+require "s3rsync/base"
+
+module S3rsync
+  
+end
+

data/s3rsync.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 's3rsync/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "s3rsync"
+  spec.version       = S3rsync::VERSION
+  spec.authors       = ["idevops"]
+  spec.email         = ["devops@innovid.com"]
+  spec.summary       = "Remote update protocol between local set of files and S3 storage"
+  spec.description   = "Remote update protocol allows sync files to transfer just the differences between two sets of files across the network connection between local filesystem and s3 storage"
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "thor", "0.19.1"
+  
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "bump"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "simplecov"
+  spec.add_development_dependency "growl"
+  spec.add_development_dependency "guard"
+  spec.add_development_dependency "guard-rspec"
+end

data/spec/spec_helper.rb

@@ -0,0 +1,12 @@
+require "simplecov"
+require "s3rsync"
+require "s3rsync/base"
+
+SimpleCov.start do
+  add_filter '/spec/'
+  add_filter '/bin/'
+end# if ENV["COVERAGE"]
+
+module Helpers
+  
+end

data/spec/sync_spec.rb

@@ -0,0 +1,50 @@
+require 'spec_helper'
+require 's3rsync/sync'
+
+include Helpers
+
+describe S3Rsync::Sync do
+  context 'sync is successfull' do
+    let(:opt) { {
+        'dir'         => '/test_dir',
+        's3_prefix'   => 's3rsync',
+        'config_path' => 'test_json_conf_path',
+      } }
+      
+    let(:json_conf) { {
+      'env.static.s3.bucket'      => 'test-bucket',
+      'env.static.s3.key.user'    => 'test_access_key',
+      'env.static.s3.key.secret'  => 'test_secret_key',
+      } }
+    
+    before do
+      allow(File).to receive(:read).and_return(nil)
+      allow(File).to receive(:read).with(opt['config_path']).and_return(json_conf.to_json)
+      allow_any_instance_of(S3Rsync::Sync).to receive(:pass_cli).and_return(nil)
+    end
+    
+    after do
+      #
+    end
+    
+    before(:each) do
+      #Helpers::pre_run_helper
+    end
+    
+    it "uploads the files" do
+      cmd = "s3cmd sync #{opt['dir']} s3://#{json_conf['env.static.s3.bucket']}/#{opt['s3_prefix']}/"
+      allow_any_instance_of(S3Rsync::Sync).to receive(:pass_cli).with(cmd).and_return('Success')
+      
+      sync = S3Rsync::Sync.new(opt)
+      expect(sync.run(:upload)).to eq 0
+    end
+    
+    it "download the files" do
+      cmd = "s3cmd sync s3://#{json_conf['env.static.s3.bucket']}/#{opt['s3_prefix']}/ #{opt['dir']}"
+      allow_any_instance_of(S3Rsync::Sync).to receive(:pass_cli).with(cmd).and_return('Success')
+      
+      sync = S3Rsync::Sync.new(opt)
+      expect(sync.run(:download)).to eq 0
+    end
+  end
+end

metadata

@@ -0,0 +1,191 @@
+--- !ruby/object:Gem::Specification
+name: s3rsync
+version: !ruby/object:Gem::Version
+  version: 0.1.3
+platform: ruby
+authors:
+- idevops
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-08-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.19.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.19.1
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bump
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: growl
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Remote update protocol allows sync files to transfer just the differences
+  between two sets of files across the network connection between local filesystem
+  and s3 storage
+email:
+- devops@innovid.com
+executables:
+- s3rsync
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- Gemfile
+- Guardfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/s3rsync
+- lib/s3rsync.rb
+- lib/s3rsync/base.rb
+- lib/s3rsync/cli.rb
+- lib/s3rsync/sync.rb
+- lib/s3rsync/version.rb
+- s3rsync.gemspec
+- spec/spec_helper.rb
+- spec/sync_spec.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.3.0
+signing_key: 
+specification_version: 4
+summary: Remote update protocol between local set of files and S3 storage
+test_files:
+- spec/spec_helper.rb
+- spec/sync_spec.rb