s3_uploader 0.1.3 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bf2cd35df114a57623ee6363704be419cce05799
-  data.tar.gz: f5723eb35fc8d41600472d02b8bb579b6374e4f6
+  metadata.gz: 4d6399ae8ff54b55635cf668b0f6b711c907fdb1
+  data.tar.gz: b3607639a7a60dcaacdbb41963f4f7e5dddfd5bb
 SHA512:
-  metadata.gz: e55f3328934aa57f1f753e1554cdcf689b9afe943662603eeb73a6b00d226475bbdfe22cc1733e235f19a18131b99cde89d8201ac113db338a525ddd79e493c6
-  data.tar.gz: 46f61b2e5c7ee2c5709df974f4fbdce5efc44090785fb0b25932c0fa880dac620ca6820379dd8eead0fe2abfa28bc011ea02a21581ef0c502be4320f69241973
+  metadata.gz: ebb2a6aab8ddd480ea1dc3f163f92f435f0847cca276fc41dbc937848827690d44e092e4af0ac1eb52ab86c1f3a0cd3577bf6d892417eed355b03cb34b1c5391
+  data.tar.gz: 81301dba831306bed8f4de9d7323509a477540c8d5da8cc12ecdc4aed5380896f821110a4f5ee824bdce88555c6cb5836ef1608c99191893900e0379adbe1489

data/README.md

@@ -30,7 +30,21 @@ Or install it yourself as:
 ## Usage
 
 ```ruby
-	S3Uploader.upload_directory('/tmp/test', 'mybucket',
+	uploader = S3Uploader::Uploader.new({
+         :s3_key => YOUR_KEY,
+         :s3_secret => YOUR_SECRET_KEY,
+         :destination_dir => 'test/',
+         :region => 'eu-west-1',
+         :threads => 10
+		})
+
+  uploader.upload('/tmp/test', 'mybucket')
+```
+
+or
+
+```ruby
+	S3Uploader.upload('/tmp/test', 'mybucket',
 		{ 	 :s3_key => YOUR_KEY,
 			   :s3_secret => YOUR_SECRET_KEY,
 			   :destination_dir => 'test/',
@@ -40,12 +54,14 @@ Or install it yourself as:
 		})
 ```
 
-If no keys are provided, it uses S3_KEY and S3_SECRET environment variables. us-east-1 is the default region.
+Former static method upload_directory is still supported for backwards compatibility.
 
 ```ruby
 	S3Uploader.upload_directory('/tmp/test', 'mybucket', { :destination_dir => 'test/', :threads => 4 })
 ```
 
+If no keys are provided, it uses S3_KEY and S3_SECRET environment variables. us-east-1 is the default region.
+
 Metadata headers are documented [here](http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html)
 
 Or as a command line binary
@@ -72,10 +88,6 @@ directory is required in this case.
     })
 ```
 
-## TODO
-
-1. Add optional time, size and number of files uploaded report at end of process
-
 ## Contributing
 
 1. Fork it
@@ -91,6 +103,7 @@ directory is required in this case.
 * [Philip Cunningham](https://github.com/unsymbol)
 * [Ludwig Bratke](https://github.com/bratke)
 * [John Pignata](https://github.com/jpignata)
+* [eperezks](https://github.com/eperezks)
 
 ## License
 

data/lib/s3_uploader/s3_uploader.rb

@@ -1,135 +1,159 @@
 module S3Uploader
   KILO_SIZE = 1024.0
   BLOCK_SIZE = 1024 * 1024
+  DEFAULT_THREADS_NUMBER = 5
+  DEFAULT_AWS_REGION = 'us-east-1'
+
+  def self.upload(source, bucket, options = {})
+     Uploader.new(options).upload(source, bucket)
+  end
 
   def self.upload_directory(source, bucket, options = {})
-    options = {
-      :destination_dir => '',
-      :threads => 5,
-      :s3_key => ENV['S3_KEY'],
-      :s3_secret => ENV['S3_SECRET'],
-      :public => false,
-      :region => 'us-east-1',
-      :metadata => {},
-      :path_style => false,
-      :regexp => /.*/,
-      :gzip => false,
-      :gzip_working_dir => nil,
-      :time_range => Time.at(0)..(Time.now + (60 * 60 * 24))
-    }.merge(options)
-
-    log = options[:logger] || Logger.new(STDOUT)
-
-    raise 'Source must be a directory' unless File.directory?(source)
-
-
-    if options[:gzip]
-      if options[:gzip_working_dir].nil?
+    self.upload(source, bucket, options)
+  end
+
+  class Uploader
+    attr_writer :logger
+
+    def initialize(options = {})
+
+      @options = {
+        :destination_dir => '',
+        :threads => DEFAULT_THREADS_NUMBER,
+        :s3_key => ENV['S3_KEY'],
+        :s3_secret => ENV['S3_SECRET'],
+        :public => false,
+        :region => DEFAULT_AWS_REGION,
+        :metadata => {},
+        :path_style => false,
+        :regexp => nil,
+        :gzip => false,
+        :gzip_working_dir => nil,
+        :time_range => Time.at(0)..(Time.now + (60 * 60 * 24))
+      }.merge(options)
+
+      @logger = @options[:logger] || Logger.new(STDOUT)
+
+      if @options[:gzip] && @options[:gzip_working_dir].nil?
         raise 'gzip_working_dir required when using gzip'
-      else
-        source_dir = source.end_with?('/') ? source : [ source, '/'].join
-        gzip_working_dir = options[:gzip_working_dir].end_with?('/') ?
-                              options[:gzip_working_dir] : [ options[:gzip_working_dir], '/'].join
+      end
 
-        if gzip_working_dir.start_with?(source_dir)
-          raise 'gzip_working_dir may not be located within source-folder'
+      if @options[:connection]
+        @connection = @options[:connection]
+      else
+        if @options[:s3_key].nil? || @options[:s3_secret].nil?
+          raise "Missing access keys"
         end
+
+        @connection = Fog::Storage.new({
+            :provider => 'AWS',
+            :aws_access_key_id => @options[:s3_key],
+            :aws_secret_access_key => @options[:s3_secret],
+            :region => @options[:region],
+            :path_style => @options[:path_style]
+        })
       end
 
-      options[:gzip_working_dir] = options[:gzip_working_dir].chop if options[:gzip_working_dir].end_with?('/')
+      if !@options[:destination_dir].to_s.empty? &&
+          !@options[:destination_dir].end_with?('/')
+        @options[:destination_dir] << '/'
+      end
     end
 
+    def upload(source, bucket)
+      raise 'Source directory is requiered' if source.to_s.empty?
+      source << '/' unless source.end_with?('/')
+      raise 'Source must be a directory' unless File.directory?(source)
 
-    if options[:connection]
-      connection = options[:connection]
-    else
-      raise "Missing access keys" if options[:s3_key].nil? || options[:s3_secret].nil?
+      gzip_working_dir = @options[:gzip_working_dir]
 
-      connection = Fog::Storage.new({
-          :provider => 'AWS',
-          :aws_access_key_id => options[:s3_key],
-          :aws_secret_access_key => options[:s3_secret],
-          :region => options[:region],
-          :path_style => options[:path_style]
-      })
-    end
+      if @options[:gzip] && !gzip_working_dir.to_s.empty?
+        gzip_working_dir << '/' unless gzip_working_dir.end_with?('/')
 
-    source = source.chop if source.end_with?('/')
+        if gzip_working_dir.start_with?(source)
+          raise 'gzip_working_dir may not be located within source-folder'
+        end
+      end
 
-    if options[:destination_dir] != '' && !options[:destination_dir].end_with?('/')
-      options[:destination_dir] = "#{options[:destination_dir]}/"
-    end
-    total_size = 0
-    files = Queue.new
-
-    Dir.glob("#{source}/**/*").select { |f| !File.directory?(f) }.each do |f|
-      if File.basename(f).match(options[:regexp]) && options[:time_range].cover?(File.mtime(f))
-        if options[:gzip] && File.extname(f) != '.gz'
-          dir, base = File.split(f)
-          dir       = dir.sub(source, options[:gzip_working_dir])
-          gz_file   = "#{dir}/#{base}.gz"
-
-          FileUtils.mkdir_p(dir) unless File.directory?(dir)
-          Zlib::GzipWriter.open(gz_file) do |gz|
-            gz.mtime     = File.mtime(f)
-            gz.orig_name = f
-
-            File.open(f, 'rb') do |fi|
-              while (block_in = fi.read(BLOCK_SIZE)) do
-                gz.write block_in
+      total_size = 0
+      files = Queue.new
+      regexp = @options[:regexp]
+      Dir.glob(File.join(source, '**/*'))
+        .select { |f| !File.directory?(f) }.each do |f|
+
+        if (regexp.nil? || File.basename(f).match(regexp)) &&
+            @options[:time_range].cover?(File.mtime(f))
+          if @options[:gzip] && File.extname(f) != '.gz'
+            dir, base = File.split(f)
+            dir       = dir.sub(source, gzip_working_dir)
+            gz_file   = File.join(dir, [ base, '.gz' ].join)
+
+            @logger.info("Compressing #{f}")
+
+            FileUtils.mkdir_p(dir)
+            Zlib::GzipWriter.open(gz_file) do |gz|
+              gz.mtime     = File.mtime(f)
+              gz.orig_name = f
+
+              File.open(f, 'rb') do |fi|
+                while (block_in = fi.read(BLOCK_SIZE)) do
+                  gz.write block_in
+                end
               end
             end
-          end
 
-          files << gz_file
-          total_size += File.size(gz_file)
-        else
-          files << f
-          total_size += File.size(f)
+            files << gz_file
+            total_size += File.size(gz_file)
+          else
+            files << f
+            total_size += File.size(f)
+          end
         end
       end
-    end
 
-    directory = connection.directories.new(:key => bucket)
+      directory = @connection.directories.new(:key => bucket)
 
-    start = Time.now
-    total_files = files.size
-    file_number = 0
-    @mutex = Mutex.new
+      start = Time.now
+      total_files = files.size
+      file_number = 0
+      @mutex = Mutex.new
 
-    threads = []
-    options[:threads].times do |i|
-      threads[i] = Thread.new do
+      threads = []
+      @options[:threads].times do |i|
+        threads[i] = Thread.new do
 
-        until files.empty?
-          @mutex.synchronize do
-            file_number += 1
-            Thread.current["file_number"] = file_number
-          end
-          file = files.pop rescue nil
-          if file
-            key = file.gsub(source, '').gsub(options[:gzip_working_dir].to_s, '')[1..-1]
-            dest = "#{options[:destination_dir]}#{key}"
-            body = File.open(file)
-            log.info("[#{Thread.current["file_number"]}/#{total_files}] Uploading #{key} to s3://#{bucket}/#{dest}")
-
-            directory.files.create(
-              :key    => dest,
-              :body   => body,
-              :public => options[:public],
-              :metadata => options[:metadata]
-            )
-            body.close
+          until files.empty?
+            @mutex.synchronize do
+              file_number += 1
+              Thread.current["file_number"] = file_number
+            end
+            file = files.pop rescue nil
+            if file
+              key = file.sub(source, '').sub(gzip_working_dir.to_s, '')
+              dest = [ @options[:destination_dir], key ].join
+              body = File.open(file)
+              @logger.info(["[", Thread.current["file_number"], "/",
+                            total_files, "] Uploading ", key,
+                            " to s3://#{bucket}/#{dest}" ].join)
+
+              directory.files.create(
+                :key    => dest,
+                :body   => body,
+                :public => @options[:public],
+                :metadata => @options[:metadata]
+              )
+              body.close
+            end
           end
         end
       end
-    end
-    threads.each { |t| t.join }
-
-    finish = Time.now
-    elapsed = finish.to_f - start.to_f
-    mins, secs = elapsed.divmod 60.0
-    log.info("Uploaded %d (%.#{0}f KB) in %d:%04.2f" % [total_files, total_size / KILO_SIZE, mins.to_i, secs])
+      threads.each { |t| t.join }
 
+      finish = Time.now
+      elapsed = finish.to_f - start.to_f
+      mins, secs = elapsed.divmod 60.0
+      @logger.info("Uploaded %d (%.#{0}f KB) in %d:%04.2f" %
+                  [total_files, total_size / KILO_SIZE, mins.to_i, secs])
+    end
   end
 end

data/lib/s3_uploader/version.rb

@@ -1,3 +1,3 @@
 module S3Uploader
-  VERSION = "0.1.3"
+  VERSION = "0.2.0"
 end

data/spec/s3uploader_spec.rb

@@ -2,15 +2,16 @@ require 'spec_helper'
 
 describe S3Uploader do
 
+  let(:tmp_directory) do
+    File.join(Dir.tmpdir, 'test_s3_uploader')
+  end
   let(:access) do
-    %w(access.log access.log.1 access.log.2.gz subdir/access.log subdir/access.log.1 subdir/access.log.2.gz)
+    %w(access.log access.log.1 access.log.2.gz subdir/access.log subdir/access.log.1 subdir/access.log.2.gz) +
+    [ File.join('subdirX', tmp_directory, 'somefile-access.txt')]
   end
   let(:error) do
     %w(error.log error.log.1 error.log.2.gz subdir/error.log subdir/error.log.1 subdir/error.log.2.gz)
   end
-  let(:tmp_directory) do
-    File.join(Dir.tmpdir, 'test_s3_uploader')
-  end
   let(:logger) do
     Logger.new(STDOUT)
   end
@@ -45,7 +46,7 @@ describe S3Uploader do
 
   it 'when called with missing access keys it should raise an exception' do
     lambda {
-      S3Uploader.upload_directory('/tmp', 'mybucket',
+      S3Uploader.upload('/tmp', 'mybucket',
                                   { destination_dir: 'test1/',
                                     s3_key:          nil,
                                     s3_secret:       nil })
@@ -54,13 +55,26 @@ describe S3Uploader do
 
   it 'when called with source not directory it should raise an exception' do
     lambda {
-      S3Uploader.upload_directory('/xzzaz1232', 'mybucket')
+      S3Uploader.upload('/xzzaz1232', 'mybucket')
     }.should raise_error('Source must be a directory')
   end
 
   it 'should upload all files in a directory' do
     connection.directories.get('mybucket', prefix: 'test1/').files.empty?.should be_true
 
+    S3Uploader.upload(tmp_directory, 'mybucket',
+                                { destination_dir: 'test1/',
+                                  logger:          logger,
+                                  connection:      connection })
+
+    files = connection.directories.get('mybucket', prefix: 'test1/').files
+    expect(files).to have((access + error).size).items
+    expect(files.map(&:key)).to match_array((access + error).map { |f| File.join('test1/', f) })
+  end
+
+  it 'should still support upload_directory static method for backwards compatibility' do
+    connection.directories.get('mybucket', prefix: 'test1/').files.empty?.should be_true
+
     S3Uploader.upload_directory(tmp_directory, 'mybucket',
                                 { destination_dir: 'test1/',
                                   logger:          logger,
@@ -75,7 +89,7 @@ describe S3Uploader do
 
     it 'should upload specific files' do
 
-      S3Uploader.upload_directory(tmp_directory, 'mybucket',
+      S3Uploader.upload(tmp_directory, 'mybucket',
                                   { logger:     logger,
                                     connection: connection,
                                     regexp:     /access/ })
@@ -91,7 +105,7 @@ describe S3Uploader do
 
     it "should require a gzip working directory" do
       lambda {
-        S3Uploader.upload_directory('/tmp', 'mybucket',
+        S3Uploader.upload('/tmp', 'mybucket',
                                     { logger:     logger,
                                       connection: connection,
                                       gzip: true })
@@ -103,7 +117,7 @@ describe S3Uploader do
       FileUtils.mkdir_p working_dir
       FileUtils.rm_rf(Dir.glob(File.join(working_dir, '*')))
 
-      S3Uploader.upload_directory(tmp_directory, 'mybucket',
+      S3Uploader.upload(tmp_directory, 'mybucket',
                                   { logger:           logger,
                                     connection:       connection,
                                     regexp:           /error/,
@@ -117,13 +131,13 @@ describe S3Uploader do
 
     it 'when called with bad gzip_working_dir it should raise an exception' do
       expect {
-        S3Uploader.upload_directory(tmp_directory, 'mybucket',
+        S3Uploader.upload(tmp_directory, 'mybucket',
                                     { gzip:             true,
                                       gzip_working_dir: File.join(tmp_directory, 'working_dir') })
       }.to raise_error('gzip_working_dir may not be located within source-folder')
 
       expect {
-        S3Uploader.upload_directory(tmp_directory, 'mybucket',
+        S3Uploader.upload(tmp_directory, 'mybucket',
                                     { logger:           logger,
                                       connection:       connection,
                                       regexp:           /non_matching/,
@@ -141,7 +155,7 @@ describe S3Uploader do
       FileUtils.mkdir_p  big_file_dir
       create_test_file(File.join(big_file_dir, 'test_big_file.dmp'), 2*1024)
 
-      S3Uploader.upload_directory(big_file_dir, 'mybucket',
+      S3Uploader.upload(big_file_dir, 'mybucket',
                                   { logger:          logger,
                                     connection:      connection,
                                     gzip:             true,
@@ -162,7 +176,7 @@ describe S3Uploader do
       yesterday  = Time.now - (60 * 60 * 24)
       File.utime(yesterday, yesterday, *file_names)
 
-      S3Uploader.upload_directory(tmp_directory, 'mybucket',
+      S3Uploader.upload(tmp_directory, 'mybucket',
                                   { logger:     logger,
                                     connection: connection,
                                     regexp:     /access/,
@@ -178,7 +192,7 @@ describe S3Uploader do
       File.utime(yesterday, yesterday, *file_names)
 
 
-      S3Uploader.upload_directory(tmp_directory, 'mybucket',
+      S3Uploader.upload(tmp_directory, 'mybucket',
                                   { logger:     logger,
                                     connection: connection,
                                     regexp:     /access/,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: s3_uploader
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - Christian Hein
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-01 00:00:00.000000000 Z
+date: 2016-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fog-aws