s3_uploader 0.1.0 → 0.1.1

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    MDZiOTVmMWI5NTExMmQ2NzEyMTA1MTliYTVhNGJhMDA1MWJmYjEwZA==
-  data.tar.gz: !binary |-
-    ZDcyZTAwNjAwOGFlNzJjYzk0YjU3MGJmYTBiNTM2MmViZDVkYTkwNQ==
+SHA1:
+  metadata.gz: 1a8fb8f3417ec607cb24ed10345b5592ddeecde7
+  data.tar.gz: 7f749a8deb7fe99d35c4365f264b2bb57206807c
 SHA512:
-  metadata.gz: !binary |-
-    YjYxNGVjODAwNjY2Y2E4YzU5NmY4NWEzNDFlNjg2NmI5NmJjYzhkYmNhMTg0
-    NWNjOGM0MzQ2Yzk4NWRiYWY5N2NkYjUyY2M2ZGUzMGZkYTYyMzEyYmFlOGMw
-    ZTI1NmQ1ZjFmYTFkN2MyNDViMWM4ZTFiMzFkNzEwYzAxNTBhZGY=
-  data.tar.gz: !binary |-
-    MGQ4NjEyMDA3N2M4OWY0NmIwMTI1MjFhY2UzY2NlY2VlZjNhM2NkNDNlZTQz
-    NmVlNTk0MzNjYjBiOWUzYjBlYTA3ZDBmMzVhZDJiMjlkN2NjYzE5Y2ZjZWFl
-    ZjgxYjA1OGVhZmU0MjFjODc3ZTE2ZDAyMmIxYjU4MzY1ZTIzMTg=
+  metadata.gz: aaaaf8f47a0548a371c867c0de00f639ec0fad740844892d4ba7d0aac4afb3679cb7fcc43737a155c72882e7212d242dfa8595d1cdd03737a705efe1d1c63b9b
+  data.tar.gz: 13a457b7691ad2a21ec201a5a64f80a36950edace655852122dc21d55e8c6128c00ec3eb0dabca051c013649b0bc1b4a4fa1d4306470894d19519a4b8cffb2f9

data/.rspec

@@ -0,0 +1,2 @@
+# Skip all tests tagged with "slow"
+--tag ~slow

data/README.md

@@ -29,18 +29,24 @@ Or install it yourself as:
 
 ## Usage
 
+```ruby
 	S3Uploader.upload_directory('/tmp/test', 'mybucket',
-		{ 	:s3_key => YOUR_KEY,
-			:s3_secret => YOUR_SECRET_KEY,
-			:destination_dir => 'test/',
-			:region => 'eu-west-1',
-			:threads => 4,
-      		:metadata => { 'Cache-Control' => 'max-age=315576000' } 
+		{ 	 :s3_key => YOUR_KEY,
+			   :s3_secret => YOUR_SECRET_KEY,
+			   :destination_dir => 'test/',
+			   :region => 'eu-west-1',
+			   :threads => 4,
+			   :metadata => { 'Cache-Control' => 'max-age=315576000' }
 		})
+```
 
 If no keys are provided, it uses S3_KEY and S3_SECRET environment variables. us-east-1 is the default region.
 
+```ruby
 	S3Uploader.upload_directory('/tmp/test', 'mybucket', { :destination_dir => 'test/', :threads => 4 })
+```
+
+Metadata headers are documented [here](http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html)
 
 Or as a command line binary
 
@@ -49,14 +55,26 @@ Or as a command line binary
 Again, it uses S3_KEY and S3_SECRET environment variables if non provided in parameters.
 
 	s3uploader -d test/ -t 4 /tmp/test mybucket
-	
-	
-Metadata headers are documented [here](http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html)
+
+## Compress files
+
+If the `:gzip` options is used, files not already compressed are packed using GZip before upload. A GZip working
+directory is required in this case.
+
+```ruby
+  S3Uploader.upload_directory('/tmp/test', 'mybucket',
+    {    :s3_key => YOUR_KEY,
+         :s3_secret => YOUR_SECRET_KEY,
+         :destination_dir => 'test/',
+         :region => 'eu-west-1',
+         :gzip => true,
+         :gzip_working_dir => '/tmp/gzip_working_dir'
+    })
+```
 
 ## TODO
 
-1. Allow regex pattern matching to select files to upload
-2. Add optional time, size and number of files uploaded report at end of process
+1. Add optional time, size and number of files uploaded report at end of process
 
 ## Contributing
 

data/lib/s3_uploader/s3_uploader.rb

@@ -1,5 +1,7 @@
 module S3Uploader
   KILO_SIZE = 1024.0
+  BLOCK_SIZE = 1024 * 1024
+
   def self.upload_directory(source, bucket, options = {})
     options = {
       :destination_dir => '',
@@ -12,21 +14,36 @@ module S3Uploader
       :path_style => false,
       :regexp => /.*/,
       :gzip => false,
-      :gzip_working_dir => source,
+      :gzip_working_dir => nil,
       :time_range => Time.at(0)..(Time.now + (60 * 60 * 24))
     }.merge(options)
 
     log = options[:logger] || Logger.new(STDOUT)
 
     raise 'Source must be a directory' unless File.directory?(source)
-    if options[:gzip_working_dir] != source && options[:gzip_working_dir][source]
-      raise 'gzip_working_dir may not be located within source-folder'
+
+
+    if options[:gzip]
+      if options[:gzip_working_dir].nil?
+        raise 'gzip_working_dir required when using gzip'
+      else
+        source_dir = source.end_with?('/') ? source : [ source, '/'].join
+        gzip_working_dir = options[:gzip_working_dir].end_with?('/') ?
+                              options[:gzip_working_dir] : [ options[:gzip_working_dir], '/'].join
+
+        if gzip_working_dir.start_with?(source_dir)
+          raise 'gzip_working_dir may not be located within source-folder'
+        end
+      end
+
+      options[:gzip_working_dir] = options[:gzip_working_dir].chop if options[:gzip_working_dir].end_with?('/')
     end
 
+
     if options[:connection]
       connection = options[:connection]
     else
-      raise "Missing access keys" if options[:s3_key].nil? or options[:s3_secret].nil?
+      raise "Missing access keys" if options[:s3_key].nil? || options[:s3_secret].nil?
 
       connection = Fog::Storage.new({
           :provider => 'AWS',
@@ -38,15 +55,15 @@ module S3Uploader
     end
 
     source = source.chop if source.end_with?('/')
-    options[:gzip_working_dir] = options[:gzip_working_dir].chop if options[:gzip_working_dir].end_with?('/')
-    if options[:destination_dir] != '' and !options[:destination_dir].end_with?('/')
+
+    if options[:destination_dir] != '' && !options[:destination_dir].end_with?('/')
       options[:destination_dir] = "#{options[:destination_dir]}/"
     end
     total_size = 0
     files = Queue.new
 
     Dir.glob("#{source}/**/*").select { |f| !File.directory?(f) }.each do |f|
-      if File.basename(f).match(options[:regexp]) and options[:time_range].cover?(File.mtime(f))
+      if File.basename(f).match(options[:regexp]) && options[:time_range].cover?(File.mtime(f))
         if options[:gzip] && File.extname(f) != '.gz'
           dir, base = File.split(f)
           dir       = dir.sub(source, options[:gzip_working_dir])
@@ -56,8 +73,14 @@ module S3Uploader
           Zlib::GzipWriter.open(gz_file) do |gz|
             gz.mtime     = File.mtime(f)
             gz.orig_name = f
-            gz.write IO.binread(f)
+
+            File.open(f, 'rb') do |fi|
+              while (block_in = fi.read(BLOCK_SIZE)) do
+                gz.write block_in
+              end
+            end
           end
+
           files << gz_file
           total_size += File.size(gz_file)
         else
@@ -76,7 +99,7 @@ module S3Uploader
 
     threads = []
     options[:threads].times do |i|
-      threads[i] = Thread.new {
+      threads[i] = Thread.new do
 
         until files.empty?
           @mutex.synchronize do
@@ -85,7 +108,7 @@ module S3Uploader
           end
           file = files.pop rescue nil
           if file
-            key = file.gsub(source, '').gsub(options[:gzip_working_dir], '')[1..-1]
+            key = file.gsub(source, '').gsub(options[:gzip_working_dir].to_s, '')[1..-1]
             dest = "#{options[:destination_dir]}#{key}"
             log.info("[#{Thread.current["file_number"]}/#{total_files}] Uploading #{key} to s3://#{bucket}/#{dest}")
 
@@ -97,7 +120,7 @@ module S3Uploader
             )
           end
         end
-      }
+      end
     end
     threads.each { |t| t.join }
 

data/lib/s3_uploader/version.rb

@@ -1,3 +1,3 @@
 module S3Uploader
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/spec/s3uploader_spec.rb

@@ -15,15 +15,31 @@ describe S3Uploader do
     Logger.new(STDOUT)
   end
 
-  before(:each) do
+  let(:connection) do
     Fog.mock!
 
-    FileUtils.rm_rf(Dir.glob("#{tmp_directory}/*"))
+    connection = Fog::Storage.new({
+      :provider                 => 'AWS',
+      :aws_access_key_id        => '11111111111',
+      :aws_secret_access_key    => 'XXXXXXXXXXXXXXXXXXXXXXXXXXX'
+    })
+
+  end
+
+  before(:each) do
+    Fog::Mock.reset
+
+    connection.directories.create(
+      :key    => 'mybucket',
+      :public => true
+    )
+
+    FileUtils.rm_rf(Dir.glob(File.join(tmp_directory, '*')))
 
     (access + error).each do |file|
-      directory, basename = File.split("#{tmp_directory}/#{file}")
+      directory, basename = File.split(File.join(tmp_directory, file))
       FileUtils.mkdir_p directory
-      Open3.popen3("dd if=/dev/zero of=#{directory}/#{basename} count=1024 bs=1024")
+      create_test_file(File.join(directory, basename), 1)
     end
   end
 
@@ -43,75 +59,49 @@ describe S3Uploader do
   end
 
   it 'should upload all files in a directory' do
-    connection = double(:connection)
-    connection.stub_chain(:directories, :new).and_return(directory = double(:directory))
-    directory.stub(:files).and_return(files = double(:files))
-
-    files.should_receive(:create).exactly(12).times
+    connection.directories.get('mybucket', prefix: 'test1/').files.empty?.should be_true
 
     S3Uploader.upload_directory(tmp_directory, 'mybucket',
                                 { destination_dir: 'test1/',
                                   logger:          logger,
                                   connection:      connection })
+
+    files = connection.directories.get('mybucket', prefix: 'test1/').files
+    expect(files).to have((access + error).size).items
+    expect(files.map(&:key)).to match_array((access + error).map { |f| File.join('test1/', f) })
   end
 
   describe 'regexp' do
 
     it 'should upload specific files' do
-      connection = double(:connection)
-      connection.stub_chain(:directories, :new).and_return(directory = double(:directory))
-      directory.stub(:files).and_return(files = double(:files))
-
-      keys = access.dup
-      files.should_receive(:create).exactly(6).times do |hash|
-        expect(keys).to include(hash[:key])
-        keys.delete(hash[:key])
-      end
 
       S3Uploader.upload_directory(tmp_directory, 'mybucket',
                                   { logger:     logger,
                                     connection: connection,
                                     regexp:     /access/ })
+
+      files = connection.directories.get('mybucket').files
+      expect(files).to have(access.size).items
+      expect(files.map(&:key)).to match_array(access)
     end
 
   end
 
   describe 'gzip' do
 
-    it 'should upload compressed files' do
-      connection = double(:connection)
-      connection.stub_chain(:directories, :new).and_return(directory = double(:directory))
-      directory.stub(:files).and_return(files = double(:files))
-
-      #expect to upload gz-files only
-      keys = error.map { |f| f.sub('.gz', '') }.map { |f| f + '.gz' }
-      files.should_receive(:create).exactly(6).times do |hash|
-        expect(keys).to include(hash[:key])
-        keys.delete(hash[:key])
-      end
-
-      S3Uploader.upload_directory(tmp_directory, 'mybucket',
-                                  { logger:     logger,
-                                    connection: connection,
-                                    regexp:     /error/,
-                                    gzip:       true })
+    it "should require a gzip working directory" do
+      lambda {
+        S3Uploader.upload_directory('/tmp', 'mybucket',
+                                    { logger:     logger,
+                                      connection: connection,
+                                      gzip: true })
+      }.should raise_error('gzip_working_dir required when using gzip')
     end
 
-    it 'should use gzip_working_dir correctly' do
+    it 'should compress files before upload when needed' do
       working_dir = File.join(Dir.tmpdir, 's3uploader_spec/working_dir')
       FileUtils.mkdir_p working_dir
-      FileUtils.rm_rf(Dir.glob("#{working_dir}/*"))
-
-      connection = double(:connection)
-      connection.stub_chain(:directories, :new).and_return(directory = double(:directory))
-      directory.stub(:files).and_return(files = double(:files))
-
-      #expect to upload gz-files only
-      keys = error.map { |f| f.sub('.gz', '') }.map { |f| f + '.gz' }
-      files.should_receive(:create).exactly(6).times do |hash|
-        expect(keys).to include(hash[:key])
-        keys.delete(hash[:key])
-      end
+      FileUtils.rm_rf(Dir.glob(File.join(working_dir, '*')))
 
       S3Uploader.upload_directory(tmp_directory, 'mybucket',
                                   { logger:           logger,
@@ -120,70 +110,83 @@ describe S3Uploader do
                                     gzip:             true,
                                     gzip_working_dir: working_dir })
 
-      #only compress files which aren't compressed yet
-      compressed_files = error.select { |f| File.extname(f) != '.gz' }.map { |f| f + '.gz' }
-      working_dir_content = Dir["#{working_dir}/**/*"].map { |f| f.sub(working_dir, '')[1..-1] }
-
-      #expect compressed files within working_directory
-      expect(working_dir_content & compressed_files).to match_array(compressed_files)
+      files = connection.directories.get('mybucket').files
+      expect(files).to have(error.size).items
+      expect(files.map(&:key)).to match_array(error.map { |f| File.extname(f) != '.gz' ? [f, '.gz'].join : f })
     end
 
     it 'when called with bad gzip_working_dir it should raise an exception' do
       expect {
         S3Uploader.upload_directory(tmp_directory, 'mybucket',
                                     { gzip:             true,
-                                      gzip_working_dir: File.join(Dir.tmpdir, 'test_s3_uploader/working_dir') })
+                                      gzip_working_dir: File.join(tmp_directory, 'working_dir') })
       }.to raise_error('gzip_working_dir may not be located within source-folder')
 
       expect {
         S3Uploader.upload_directory(tmp_directory, 'mybucket',
-                                    { gzip:             true,
+                                    { logger:           logger,
+                                      connection:       connection,
+                                      regexp:           /non_matching/,
+                                      gzip:             true,
                                       gzip_working_dir: File.join(Dir.tmpdir, 'test_s3_uploader_working_dir') })
-      }.to raise_error('gzip_working_dir may not be located within source-folder')
+      }.to_not raise_error
+
     end
 
+    # Run with: rspec --tag slow
+    it 'uploads large files', :slow do
+      working_dir = File.join(Dir.tmpdir, 's3uploader_big_file_spec/working_dir')
+      big_file_dir = File.join(Dir.tmpdir, 'test_s3_uploader_big_file')
+      FileUtils.mkdir_p working_dir
+      FileUtils.mkdir_p  big_file_dir
+      create_test_file(File.join(big_file_dir, 'test_big_file.dmp'), 2*1024)
+
+      S3Uploader.upload_directory(big_file_dir, 'mybucket',
+                                  { logger:          logger,
+                                    connection:      connection,
+                                    gzip:             true,
+                                    gzip_working_dir: working_dir })
+
+      files = connection.directories.get('mybucket').files
+      expect(files.map(&:key)).to match_array([ 'test_big_file.dmp.gz' ])
+
+      FileUtils.rm_rf(Dir.glob(File.join(working_dir, '*')))
+      FileUtils.rm_rf(Dir.glob(File.join(big_file_dir, '*')))
+    end
   end
 
   describe 'time_range' do
 
     it 'should not upload any files' do
-      connection = double(:connection)
-      connection.stub_chain(:directories, :new).and_return(directory = double(:directory))
-      directory.stub(:files).and_return(files = double(:files))
-
-      file_names = access.map { |f| "#{tmp_directory}/#{f}" }
+      file_names = access.map { |f| File.join( tmp_directory, f) }
       yesterday  = Time.now - (60 * 60 * 24)
       File.utime(yesterday, yesterday, *file_names)
 
-      files.should_not_receive(:create)
-
       S3Uploader.upload_directory(tmp_directory, 'mybucket',
                                   { logger:     logger,
                                     connection: connection,
                                     regexp:     /access/,
                                     time_range: (Time.now - (60 * 60 * 12))..Time.now })
+
+      files = connection.directories.get('mybucket').files
+      expect(files).to have(0).items
     end
 
     it 'should upload files' do
-      connection = double(:connection)
-      connection.stub_chain(:directories, :new).and_return(directory = double(:directory))
-      directory.stub(:files).and_return(files = double(:files))
-
-      file_names = access.map { |f| "#{tmp_directory}/#{f}" }
+      file_names = access.map { |f| File.join( tmp_directory, f) }
       yesterday  = Time.now - (60 * 60 * 12)
       File.utime(yesterday, yesterday, *file_names)
 
-      keys = access.dup
-      files.should_receive(:create).exactly(6).times do |hash|
-        expect(keys).to include(hash[:key])
-        keys.delete(hash[:key])
-      end
 
       S3Uploader.upload_directory(tmp_directory, 'mybucket',
                                   { logger:     logger,
                                     connection: connection,
                                     regexp:     /access/,
                                     time_range: (Time.now - (60 * 60 * 24))..Time.now })
+
+      files = connection.directories.get('mybucket').files
+      expect(files).to have(access.size).items
+      expect(files.map(&:key)).to match_array(access)
     end
 
   end

data/spec/spec_helper.rb

@@ -7,4 +7,13 @@ require 'open3'
 RSpec.configure do |config|
   config.color_enabled = true
   config.formatter = 'documentation'
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+end
+
+
+def create_test_file(filename, size)
+  File.open(filename, 'w') do |f|
+    contents = "x" * (1024*1024)
+    size.to_i.times { f.write(contents) }
+  end
 end

metadata

@@ -1,55 +1,55 @@
 --- !ruby/object:Gem::Specification
 name: s3_uploader
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Christian Hein
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-15 00:00:00.000000000 Z
+date: 2014-08-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fog
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 2.14.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 2.14.1
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: S3 multithreaded directory uploader
@@ -60,7 +60,8 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
+- ".rspec"
 - Gemfile
 - LICENSE
 - README.md
@@ -82,17 +83,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.1.2
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: S3 multithreaded directory uploader