s3rsync 0.1.4 → 0.1.5

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MzYzODU3N2U1OGFmNzEwNzFlNzRkZWRkMmQ2YTZhY2ZlZTNkM2QyMw==
+    MDUyOTRiOWFhOWJjNmY4ZjVkZWI0NzFkY2IwNWFlM2EzNjU5Yzk2Zg==
   data.tar.gz: !binary |-
-    YjhlMjJjZmFmNThhNzRkYjYwNWNmNzdjODc2N2Q2OWQwODY2NGE4OQ==
+    MTg4ZjgzMDA0YzgxMDM5NmQ4YjIzZDdhMmRhZmYzNzRmYjhjMGY5OA==
 SHA512:
   metadata.gz: !binary |-
-    NTAzOGRlMTc5NWVlMjg5ZGNkNWNhMDVhMzRlNmViYjNmMTIzZGQ3ZmM2MTRl
-    ZjA5YzMyYzBiNjI2Mzk2OTY1ZjVlYzcwOTZiNzdkYTY2MzYzYmZiMGQ2YjU0
-    ODdmZGFkNjY1NDM3NjhhOTU0NjFjODMzOTVhZTVjNzYxOTY0MzU=
+    ZTM3OGY2NWQzMTVmM2RjN2U2NGZhYmM3Njc1YjIwODM4ZGI4MmNlZDc4YTQw
+    YWVlMjBlOTcwYjdhYzI1NzY3ZDY4OGQ1YWRkZTRiMzI0NDYxNDQ2OWY2YmM4
+    NDEwMGZiNDQ2OGFjN2I5NTE2MTc4Mjg1ZWExYzg4NDVhY2QwMWM=
   data.tar.gz: !binary |-
-    N2U1OGI0OWEyNDIwOWNlMDkwOTljMDVmMTRlMTBlMzM4ZTY4ZGI0ZDk3MTdi
-    YWNkMzQ1ZmMwYmY5NGIxN2I0NmE3N2NjZmMyODIwZjE0MzQxMzVmMWIyZDY2
-    MmEyNzhlMGIxMmEwYmQ0ODQ2MjU5YjY5OWYyMGZiZjFkYzBkZmE=
+    YmIzZGEwMzZiZmU1NmE4ZmMzNTBmMWE4ZWIxYjFjZWE3MTg3ZmQ1YTBhNjhk
+    ZGJlOGZkMjkxM2EzYzU5ZWM5YTNlNmMwNjBjNTNhMzJkNDU3N2RlNjA0NWRl
+    M2UyZmEzOTEwOTlmZjVlMmVhMTAyNjNlNGU2ZjcyZWY0YzZiZWU=

data/lib/s3rsync/base.rb

@@ -4,8 +4,9 @@ require 'socket'
 
 module S3Rsync
   class Base
-    def init_logger
-      logger = Logger.new(STDOUT)
+    def init_logger(log_path = nil)
+      output_file = log_path.nil? ? STDOUT : log_path
+      logger = Logger.new(output_file)
       logger.level = (!ENV['DEBUG'].nil?) ? Logger::DEBUG : Logger::INFO
       logger.debug "logger.level: #{logger.level}"
       logger

data/lib/s3rsync/cli.rb

@@ -2,29 +2,31 @@ require 'thor'
 
 module S3Rsync
   class CLI < Thor
-    DEFAULT_SETTINGS = {
-      :lock_file_path => "default",
-    }
-
-    desc "upload", "Upload"
-    method_option :dir, :aliases => "-d", :required => true
-    method_option :config_path, :aliases => "-c", :required => true
-    method_option :s3_prefix, :aliases => "-p", :default => 's3rsync'
+    
+    def self.common_opts
+      method_option :path, :aliases => "-p", :required => true
+      method_option :'config-path', :aliases => "-c", :required => true
+      method_option :'s3-prefix', :aliases => "-s", :default => 's3rsync'
+      method_option :'log-path', :aliases => "-l"
+      method_option :'dry-run', :aliases => "-d", :type => :boolean, :default => false
+    end
+    
+    desc "upload", "sync from local to s3"
+    common_opts
     def upload
       require 's3rsync/sync'
       sync = Sync.new(options)
-      sync.run(:upload)
+      exit sync.run(:upload)
     end
     
-    desc "download", "Download"
-    method_option :dir, :aliases => "-d", :required => true
-    method_option :config_path, :aliases => "-c", :required => true
-    method_option :s3_prefix, :aliases => "-p", :default => 's3rsync'
+    desc "download", "sync from s3 to local"
+    common_opts
     def download
       require 's3rsync/sync'
       sync = Sync.new(options)
-      sync.run(:download)
+      exit sync.run(:download)
     end
-
+    
   end
+  
 end

data/lib/s3rsync/sync.rb

@@ -1,19 +1,19 @@
 require "s3rsync"
 require "json"
 require "lockfile"
+require "erb"
 
 module S3Rsync
   class Sync < Base
   
     def initialize(opt)
-      @log = init_logger
+      @log = init_logger(opt['log-path'])
       @log.debug "opt: #{opt.inspect}"
       @params = {}
-      @params[:dir]         = opt['dir']
-      @params[:s3_prefix]   = opt['s3_prefix']
-      @params[:config_path] = opt['config_path']
-      @s3cmd = 's3cmd'
-      
+      @params[:path]        = opt['path'].chomp('/')
+      @params[:s3_prefix]   = opt['s3-prefix']
+      @params[:config_path] = opt['config-path']
+      @params[:s3cmd_conf]  = @params[:config_path].sub('.json', '_s3rsync.conf')
       @log.debug "@params[:config_path]: #{@params[:config_path]}"
       begin
         conf = JSON.parse File.read(@params[:config_path])
@@ -23,7 +23,7 @@ module S3Rsync
         raise mgs
         exit 1
       end
-    
+      
       @log.debug "conf: #{conf}"
       @params[:s3_bucket]   = conf['env.static.s3.bucket']
       @params[:s3_access]   = conf['env.static.s3.key.user']
@@ -33,19 +33,50 @@ module S3Rsync
       @log.debug "@params: #{@params.inspect}"
       @statsd = init_statsd @params[:statsd_host], @params[:statsd_port]
       @log.debug "@statsd: #{@statsd.inspect}"
+      
+      init_s3cmd_conf
+      @s3cmd = "s3cmd -c #{@params[:s3cmd_conf]}"
     end
     
     def get_lock(name = 'common')
       lockfile = Lockfile.new(
       "/tmp/s3rsync_#{name}.lock",
-      :timeout  => 10,
+      :timeout  => 3,
       :max_age  => 3600,
       :debug    => !ENV['DEBUG'].nil?
       )
     end
+    
+    def init_s3cmd_conf
+      @log.debug "@params[:s3cmd_conf]: #{@params[:s3cmd_conf]}"
+      if !File.exists? @params[:s3cmd_conf]
+        @log.info "Generating dynamic s3cfg: #{@params[:s3cmd_conf]}"
+        begin
+          erb_path = File.join(File.dirname(__FILE__), 'templates', 's3cmd_conf.erb')
+          erb_template = IO.read erb_path
+          @log.debug "erb_template: #{erb_template.inspect}"
+          erb = ERB.new erb_template
+          rendered_template = erb.result(binding)
+          @log.debug "rendered_template: #{rendered_template}"
+          File.write @params[:s3cmd_conf], rendered_template
+        rescue Exception => e
+          @log.error "generating s3cmd config failed: #{e.inspect}"
+          exit 1
+        end
+        @log.info "Dynamic s3cfg generated: #{@params[:s3cmd_conf]}"
+      end
+    end
   
     def run(mode = :upload)
-      sync_locations = "#{@params[:dir]} s3://#{@params[:s3_bucket]}/#{@params[:s3_prefix]}/"
+      init_s3cmd_conf
+      
+      if !Dir.exists?(@params[:path])
+        @log.error "Local dir not exist on disk: #{@params[:path]}"
+        return 4
+      end
+      
+      base_dir_name = @params[:path].split('/').last
+      sync_locations = "#{@params[:path]}/ s3://#{@params[:s3_bucket]}/#{@params[:s3_prefix]}/#{base_dir_name}/"
       @log.debug "original sync_locations: #{sync_locations}"
       lockfile = get_lock(mode)
       @log.debug "lockfile: #{lockfile.inspect}"
@@ -63,7 +94,9 @@ module S3Rsync
       sync_output = 1
       begin
         lockfile.lock
-        sync_output = pass_cli "s3cmd sync #{sync_locations}"
+        cmd = "#{@s3cmd} sync #{sync_locations}"
+        @log.debug "cmd: #{cmd}"
+        sync_output = pass_cli cmd
         @log.debug "sync_output: #{sync_output}"
       rescue Exception => e
         sync_output = 1

data/lib/s3rsync/templates/s3cmd_conf.erb

@@ -0,0 +1,44 @@
+[default]
+access_key = <%= @params[:s3_access] %>
+bucket_location = US
+cloudfront_host = cloudfront.amazonaws.com
+default_mime_type = binary/octet-stream
+delete_removed = False
+dry_run = False
+enable_multipart = True
+encoding = UTF-8
+encrypt = False
+follow_symlinks = False
+force = False
+get_continue = False
+gpg_command = /usr/bin/gpg
+gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
+gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
+gpg_passphrase = <%= @gpg_passphrase %>
+guess_mime_type = True
+host_base = s3.amazonaws.com
+host_bucket = %(bucket)s.s3.amazonaws.com
+human_readable_sizes = False
+invalidate_on_cf = False
+list_md5 = False
+log_target_prefix = 
+mime_type = 
+multipart_chunk_size_mb = 15
+preserve_attrs = True
+progress_meter = True
+proxy_host = 
+proxy_port = 0
+recursive = False
+recv_chunk = 4096
+reduced_redundancy = False
+secret_key = <%= @params[:s3_secret] %>
+send_chunk = 4096
+simpledb_host = sdb.amazonaws.com
+skip_existing = False
+socket_timeout = 300
+urlencoding_mode = normal
+use_https = False
+verbosity = WARNING
+website_endpoint = http://%(bucket)s.s3-website-%(location)s.amazonaws.com/
+website_error = 
+website_index = index.html

data/lib/s3rsync/version.rb

@@ -1,3 +1,3 @@
 module S3rsync
-  VERSION = "0.1.4"
+  VERSION = "0.1.5"
 end

data/spec/sync_spec.rb

@@ -5,21 +5,30 @@ include Helpers
 
 describe S3Rsync::Sync do
   context 'sync is successfull' do
-    let(:opt) { {
-        'dir'         => '/test_dir',
-        's3_prefix'   => 's3rsync',
-        'config_path' => 'test_json_conf_path',
-      } }
+    let(:opt) { 
+      {
+        'path'        => '/parent_test_dir/test_dir',
+        's3-prefix'   => 's3rsync',
+        'config-path' => 'test_json_conf_path.json',
+      }
+    }
+    
+    let(:json_conf) { 
+      {
+        'env.static.s3.bucket'      => 'test-bucket',
+        'env.static.s3.key.user'    => 'test_access_key',
+        'env.static.s3.key.secret'  => 'test_secret_key',
+      } 
+    }
       
-    let(:json_conf) { {
-      'env.static.s3.bucket'      => 'test-bucket',
-      'env.static.s3.key.user'    => 'test_access_key',
-      'env.static.s3.key.secret'  => 'test_secret_key',
-      } }
+    let(:s3_conf_path) { opt['config-path'].sub('.json', '_s3rsync.conf') }
+    let(:base_sync_path) { opt['path'].split('/').last }
     
     before do
       allow(File).to receive(:read).and_return(nil)
-      allow(File).to receive(:read).with(opt['config_path']).and_return(json_conf.to_json)
+      allow(File).to receive(:read).with(opt['config-path']).and_return(json_conf.to_json)
+      allow(Dir).to receive(:exists?).and_return(nil)
+      allow(Dir).to receive(:exists?).with(opt['path']).and_return(true)
       allow_any_instance_of(S3Rsync::Sync).to receive(:pass_cli).and_return(nil)
     end
     
@@ -31,16 +40,20 @@ describe S3Rsync::Sync do
       #Helpers::pre_run_helper
     end
     
-    it "uploads the files" do
-      cmd = "s3cmd sync #{opt['dir']} s3://#{json_conf['env.static.s3.bucket']}/#{opt['s3_prefix']}/"
+    after(:each) do
+      File.unlink s3_conf_path
+    end
+    
+    it "uploads dir" do
+      cmd = "s3cmd -c #{s3_conf_path} sync #{opt['path']}/ s3://#{json_conf['env.static.s3.bucket']}/#{opt['s3-prefix']}/#{base_sync_path}/"
       allow_any_instance_of(S3Rsync::Sync).to receive(:pass_cli).with(cmd).and_return('Success')
       
       sync = S3Rsync::Sync.new(opt)
       expect(sync.run(:upload)).to eq 0
     end
     
-    it "download the files" do
-      cmd = "s3cmd sync s3://#{json_conf['env.static.s3.bucket']}/#{opt['s3_prefix']}/ #{opt['dir']}"
+    it "downloads dir" do
+      cmd = "s3cmd -c #{s3_conf_path} sync s3://#{json_conf['env.static.s3.bucket']}/#{opt['s3-prefix']}/#{base_sync_path}/ #{opt['path']}/"
       allow_any_instance_of(S3Rsync::Sync).to receive(:pass_cli).with(cmd).and_return('Success')
       
       sync = S3Rsync::Sync.new(opt)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: s3rsync
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - idevops
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-03 00:00:00.000000000 Z
+date: 2014-08-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -200,6 +200,7 @@ files:
 - lib/s3rsync/base.rb
 - lib/s3rsync/cli.rb
 - lib/s3rsync/sync.rb
+- lib/s3rsync/templates/s3cmd_conf.erb
 - lib/s3rsync/version.rb
 - s3rsync.gemspec
 - spec/spec_helper.rb