s3rsync 0.1.3 → 0.1.4

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MjAyMmU2ZDVkNmM1YTdlNDdjOWNhYWZjY2FhN2U4MzJiY2I2MzRmZg==
+    MzYzODU3N2U1OGFmNzEwNzFlNzRkZWRkMmQ2YTZhY2ZlZTNkM2QyMw==
   data.tar.gz: !binary |-
-    YTkzYzFjNWFkYWE1MTk3NzRjMmYxY2Y0ZjU5MWE4MTY1Yzk3NWNmNg==
+    YjhlMjJjZmFmNThhNzRkYjYwNWNmNzdjODc2N2Q2OWQwODY2NGE4OQ==
 SHA512:
   metadata.gz: !binary |-
-    N2UxNTc0ZGNmZTE5ODkyNzcwYzE1NzczMGY0ZDM0MjM4MjJmOWM0YzVhZmM4
-    YzIyM2Q5OWU0MjU3OTI0YWM0NGE1MGZlZGMzZDdlNmMxN2MzYjNmYWQ0ZjUz
-    OGFjOGZiYjJhOWU1Yzg3N2Q5Y2FiOWU1NjI2N2Y5MWE4MDBjMWE=
+    NTAzOGRlMTc5NWVlMjg5ZGNkNWNhMDVhMzRlNmViYjNmMTIzZGQ3ZmM2MTRl
+    ZjA5YzMyYzBiNjI2Mzk2OTY1ZjVlYzcwOTZiNzdkYTY2MzYzYmZiMGQ2YjU0
+    ODdmZGFkNjY1NDM3NjhhOTU0NjFjODMzOTVhZTVjNzYxOTY0MzU=
   data.tar.gz: !binary |-
-    OTE3NjU4MWQ2YWY0ZTJlNDVjMWU2Nzk0ZTk1NGRkYTlkOWYyNzQ0ZjNhMjYw
-    ZDNjMjlhYTJlYzJkNjBjNzEzOGRmNTI5YTZkYTA1YzQyNmUwNTNhYzBkN2Mx
-    NDFkMDdhODBjYWVlNmY1MDgyN2ZkYmVhNTJlMmFjMDY5MWM1MDU=
+    N2U1OGI0OWEyNDIwOWNlMDkwOTljMDVmMTRlMTBlMzM4ZTY4ZGI0ZDk3MTdi
+    YWNkMzQ1ZmMwYmY5NGIxN2I0NmE3N2NjZmMyODIwZjE0MzQxMzVmMWIyZDY2
+    MmEyNzhlMGIxMmEwYmQ0ODQ2MjU5YjY5OWYyMGZiZjFkYzBkZmE=

data/README.md

@@ -1,13 +1,23 @@
-# S3rsync
-
+s3rsync
 ============================================
 [![Gem Version](http://img.shields.io/gem/v/s3rsync.svg)][gem]
 [![Build Status](http://img.shields.io/travis/idevops/s3rsync.svg)][travis]
+[![Coverage Status](https://img.shields.io/coveralls/idevops/s3rsync.svg)][coveralls]
 
 [gem]: https://rubygems.org/gems/s3rsync
 [travis]: http://travis-ci.org/idevops/s3rsync
+[coveralls]: https://coveralls.io/r/idevops/s3rsync
+
+Wrapper gem to sync files with s3cmd tool.
+Remote update protocol between local set of files and S3 storage.
 
-Wrapper gem to sync files with s3cmd tool
+Prerequisites
+-------------
+Omnibus is designed to run with a minimal set of prerequisites. You will need the following:
+
+- Ruby 1.9+
+- Bundler
+- s3cmd
 
 ## Installation
 
@@ -25,7 +35,10 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+    $ s3rsync help [COMMAND]                                        # Describe available commands or one specific command
+    $ s3rsync upload -c, --config-path=CONFIG_PATH -d, --dir=DIR    # Sync local dir to s3 location
+    $ s3rsync download -c, --config-path=CONFIG_PATH -d, --dir=DIR  # Sync from s3 location to local dir
+
 
 ## Contributing
 

data/lib/s3rsync/base.rb

@@ -1,4 +1,6 @@
 require 'logger'
+require 'statsd'
+require 'socket'
 
 module S3Rsync
   class Base
@@ -9,6 +11,22 @@ module S3Rsync
       logger
     end
     
+    # Usage:
+    #@statsd.increment 'garets'
+    #@statsd.timing 'glork', 320
+    #@statsd.gauge 'bork', 100
+    def init_statsd(host = 'localhost', port = 9125)
+      hostname_prefix = Socket.gethostname.split('.').first
+      statsd = Statsd.new host, port
+      statsd.namespace = hostname_prefix
+      statsd
+    end
+    
+    def error_exit(msg)
+      @log.error msg
+      return 1
+    end
+    
     def error_exit(msg)
       @log.error msg
       return 1

data/lib/s3rsync/sync.rb

@@ -1,5 +1,6 @@
 require "s3rsync"
 require "json"
+require "lockfile"
 
 module S3Rsync
   class Sync < Base
@@ -24,22 +25,53 @@ module S3Rsync
       end
     
       @log.debug "conf: #{conf}"
-      @params[:s3_bucket] = conf['env.static.s3.bucket']
-      @params[:s3_access] = conf['env.static.s3.key.user']
-      @params[:s3_secret] = conf['env.static.s3.key.secret']
+      @params[:s3_bucket]   = conf['env.static.s3.bucket']
+      @params[:s3_access]   = conf['env.static.s3.key.user']
+      @params[:s3_secret]   = conf['env.static.s3.key.secret']
+      @params[:statsd_host] = conf['env.metrics.host'] || 'localhost'
+      @params[:statsd_port] = conf['env.metrics.port'] || 8125
       @log.debug "@params: #{@params.inspect}"
+      @statsd = init_statsd @params[:statsd_host], @params[:statsd_port]
+      @log.debug "@statsd: #{@statsd.inspect}"
+    end
+    
+    def get_lock(name = 'common')
+      lockfile = Lockfile.new(
+      "/tmp/s3rsync_#{name}.lock",
+      :timeout  => 10,
+      :max_age  => 3600,
+      :debug    => !ENV['DEBUG'].nil?
+      )
     end
   
     def run(mode = :upload)
       sync_locations = "#{@params[:dir]} s3://#{@params[:s3_bucket]}/#{@params[:s3_prefix]}/"
       @log.debug "original sync_locations: #{sync_locations}"
+      lockfile = get_lock(mode)
+      @log.debug "lockfile: #{lockfile.inspect}"
+      if lockfile.locked?
+        @log.warn "lock file found, exiting: #{lockfile.inspect}"
+        return 5
+      end
+      
       if mode == :download
         @log.debug "reversing sync_locations"
         sync_locations = sync_locations.split(' ').reverse.join(' ')
         @log.debug "download sync_locations: #{sync_locations}"
       end
-      sync_output = pass_cli "s3cmd sync #{sync_locations}"
-      @log.debug "sync_output: #{sync_output}"
+      
+      sync_output = 1
+      begin
+        lockfile.lock
+        sync_output = pass_cli "s3cmd sync #{sync_locations}"
+        @log.debug "sync_output: #{sync_output}"
+      rescue Exception => e
+        sync_output = 1
+        @log.error "something wrong while trying to run the sync: #{e.inspect}"
+      ensure
+        lockfile.unlock
+      end
+      
       if (sync_output == 1) || sync_output.downcase.include?('error')
         @log.error "errors found in output: #{sync_output}"
         return 1

data/lib/s3rsync/version.rb

@@ -1,3 +1,3 @@
 module S3rsync
-  VERSION = "0.1.3"
+  VERSION = "0.1.4"
 end

data/s3rsync.gemspec

@@ -19,6 +19,8 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "thor", "0.19.1"
+  spec.add_dependency "lockfile", "2.1.3"
+  spec.add_dependency "statsd-ruby", "1.2.1"
   
   spec.add_development_dependency "bundler", "~> 1.5"
   spec.add_development_dependency "rake"
@@ -28,4 +30,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "growl"
   spec.add_development_dependency "guard"
   spec.add_development_dependency "guard-rspec"
+  spec.add_development_dependency "coveralls"
 end

data/spec/spec_helper.rb

@@ -1,7 +1,13 @@
 require "simplecov"
+require "coveralls"
 require "s3rsync"
 require "s3rsync/base"
 
+SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[
+  SimpleCov::Formatter::HTMLFormatter,
+  Coveralls::SimpleCov::Formatter
+]
+
 SimpleCov.start do
   add_filter '/spec/'
   add_filter '/bin/'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: s3rsync
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - idevops
@@ -24,6 +24,34 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.19.1
+- !ruby/object:Gem::Dependency
+  name: lockfile
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.1.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.1.3
+- !ruby/object:Gem::Dependency
+  name: statsd-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.2.1
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -136,6 +164,20 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Remote update protocol allows sync files to transfer just the differences
   between two sets of files across the network connection between local filesystem
   and s3 storage