s3_uploader 0.0.9 → 0.1.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    N2JiYmFhYzUzMjEwZmUzN2RlYzM2YTQ2MGZiNWRjNDcyN2ZlYjFiOA==
+    MDZiOTVmMWI5NTExMmQ2NzEyMTA1MTliYTVhNGJhMDA1MWJmYjEwZA==
   data.tar.gz: !binary |-
-    ZDU4ZjExZWI0MjJkNDM3MzBhNTExYzExOWJiYjE3NWJmOTg3YzkzNQ==
+    ZDcyZTAwNjAwOGFlNzJjYzk0YjU3MGJmYTBiNTM2MmViZDVkYTkwNQ==
 SHA512:
   metadata.gz: !binary |-
-    MDMyNjE0ZjJlNzdjOWQ3NmI5MzYwYWM5ZDliYzAwOTQzZWE1MTY2YjU5OGY3
-    MjQ0ZDMzOGQ1N2Q5NDYzNGViYTZlMmQxNDExN2Q3ZmE3N2E2NjkzZDJhNGVh
-    Y2YxMTNlNzQ0OTNjYTEwNWFjODlkN2Y1MDRhMDM0MjcxMjE4ZDA=
+    YjYxNGVjODAwNjY2Y2E4YzU5NmY4NWEzNDFlNjg2NmI5NmJjYzhkYmNhMTg0
+    NWNjOGM0MzQ2Yzk4NWRiYWY5N2NkYjUyY2M2ZGUzMGZkYTYyMzEyYmFlOGMw
+    ZTI1NmQ1ZjFmYTFkN2MyNDViMWM4ZTFiMzFkNzEwYzAxNTBhZGY=
   data.tar.gz: !binary |-
-    ZjRhY2RhZTA1NDBmZTcxYTg4MGRmYTE3MTQ3Njk4NjZiMjVjZmMyMTY2ZDk4
-    N2I0YjJlNjMwYjZjMDNhZTIwYjFiNDFmZGRkNDYwOGI0OWFkNTdiM2RlYmE2
-    MTllYTgyMTM2MTAxYzg5MDYyZTQyNzU5ZWQ0OGFmZTA0YzgyNDM=
+    MGQ4NjEyMDA3N2M4OWY0NmIwMTI1MjFhY2UzY2NlY2VlZjNhM2NkNDNlZTQz
+    NmVlNTk0MzNjYjBiOWUzYjBlYTA3ZDBmMzVhZDJiMjlkN2NjYzE5Y2ZjZWFl
+    ZjgxYjA1OGVhZmU0MjFjODc3ZTE2ZDAyMmIxYjU4MzY1ZTIzMTg=

data/README.md

@@ -70,6 +70,8 @@ Metadata headers are documented [here](http://docs.aws.amazon.com/AmazonS3/lates
 
 * [Mark Wagner](https://github.com/theSociableme)
 * [Brandon Hilkert](https://github.com/brandonhilkert)
+* [Philip Cunningham](https://github.com/unsymbol)
+* [Ludwig Bratke](https://github.com/bratke)
 
 ## License
 

data/lib/s3_uploader/s3_uploader.rb

@@ -9,12 +9,19 @@ module S3Uploader
       :public => false,
       :region => 'us-east-1',
       :metadata => {},
-      :path_style => false
+      :path_style => false,
+      :regexp => /.*/,
+      :gzip => false,
+      :gzip_working_dir => source,
+      :time_range => Time.at(0)..(Time.now + (60 * 60 * 24))
     }.merge(options)
 
     log = options[:logger] || Logger.new(STDOUT)
 
     raise 'Source must be a directory' unless File.directory?(source)
+    if options[:gzip_working_dir] != source && options[:gzip_working_dir][source]
+      raise 'gzip_working_dir may not be located within source-folder'
+    end
 
     if options[:connection]
       connection = options[:connection]
@@ -31,15 +38,33 @@ module S3Uploader
     end
 
     source = source.chop if source.end_with?('/')
+    options[:gzip_working_dir] = options[:gzip_working_dir].chop if options[:gzip_working_dir].end_with?('/')
     if options[:destination_dir] != '' and !options[:destination_dir].end_with?('/')
       options[:destination_dir] = "#{options[:destination_dir]}/"
     end
     total_size = 0
     files = Queue.new
-    Dir.glob("#{source}/**/*").select{ |f| !File.directory?(f) }.each do |f|
-      files << f
-      total_size += File.size(f)
 
+    Dir.glob("#{source}/**/*").select { |f| !File.directory?(f) }.each do |f|
+      if File.basename(f).match(options[:regexp]) and options[:time_range].cover?(File.mtime(f))
+        if options[:gzip] && File.extname(f) != '.gz'
+          dir, base = File.split(f)
+          dir       = dir.sub(source, options[:gzip_working_dir])
+          gz_file   = "#{dir}/#{base}.gz"
+
+          FileUtils.mkdir_p(dir) unless File.directory?(dir)
+          Zlib::GzipWriter.open(gz_file) do |gz|
+            gz.mtime     = File.mtime(f)
+            gz.orig_name = f
+            gz.write IO.binread(f)
+          end
+          files << gz_file
+          total_size += File.size(gz_file)
+        else
+          files << f
+          total_size += File.size(f)
+        end
+      end
     end
 
     directory = connection.directories.new(:key => bucket)
@@ -60,7 +85,7 @@ module S3Uploader
           end
           file = files.pop rescue nil
           if file
-            key = file.gsub(source, '')[1..-1]
+            key = file.gsub(source, '').gsub(options[:gzip_working_dir], '')[1..-1]
             dest = "#{options[:destination_dir]}#{key}"
             log.info("[#{Thread.current["file_number"]}/#{total_files}] Uploading #{key} to s3://#{bucket}/#{dest}")
 

data/lib/s3_uploader/version.rb

@@ -1,3 +1,3 @@
 module S3Uploader
-  VERSION = "0.0.9"
+  VERSION = "0.1.0"
 end

data/s3_uploader.gemspec

@@ -18,6 +18,6 @@ Gem::Specification.new do |gem|
   
   gem.add_dependency 'fog'
   
-  gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'rspec', '~>2.14.1'
   gem.add_development_dependency 'rake'
 end

data/spec/s3uploader_spec.rb

@@ -1,63 +1,191 @@
 require 'spec_helper'
 
 describe S3Uploader do
-  
+
+  let(:access) do
+    %w(access.log access.log.1 access.log.2.gz subdir/access.log subdir/access.log.1 subdir/access.log.2.gz)
+  end
+  let(:error) do
+    %w(error.log error.log.1 error.log.2.gz subdir/error.log subdir/error.log.1 subdir/error.log.2.gz)
+  end
+  let(:tmp_directory) do
+    File.join(Dir.tmpdir, 'test_s3_uploader')
+  end
+  let(:logger) do
+    Logger.new(STDOUT)
+  end
+
+  before(:each) do
+    Fog.mock!
+
+    FileUtils.rm_rf(Dir.glob("#{tmp_directory}/*"))
+
+    (access + error).each do |file|
+      directory, basename = File.split("#{tmp_directory}/#{file}")
+      FileUtils.mkdir_p directory
+      Open3.popen3("dd if=/dev/zero of=#{directory}/#{basename} count=1024 bs=1024")
+    end
+  end
+
   it 'when called with missing access keys it should raise an exception' do
     lambda {
       S3Uploader.upload_directory('/tmp', 'mybucket',
-        { :destination_dir => 'test1/',
-          :s3_key => nil,
-          :s3_secret => nil
-        })
+                                  { destination_dir: 'test1/',
+                                    s3_key:          nil,
+                                    s3_secret:       nil })
     }.should raise_error('Missing access keys')
   end
-  
+
   it 'when called with source not directory it should raise an exception' do
     lambda {
       S3Uploader.upload_directory('/xzzaz1232', 'mybucket')
     }.should raise_error('Source must be a directory')
   end
-  
-  
-  before :all do
-    Fog.mock!
-    
-    @connection = Fog::Storage.new({
-      :provider                 => 'AWS',
-      :aws_access_key_id        => '11111111111',
-      :aws_secret_access_key    => 'XXXXXXXXXXXXXXXXXXXXXXXXXXX'
-    })
-    
-    @connection.directories.create(
-      :key    => 'mybucket',
-      :public => true
-    )
-    
-    @tmp_directory = File.join(Dir.tmpdir, 'test_s3_uploader')
-    create_test_files(@tmp_directory, 10)
-    create_test_files(File.join(@tmp_directory, 'subdir1'), 5)
-    @logger = Logger.new(STDOUT)
-  end
-  
-  it "should upload all files in a directory" do
-    puts @tmp_directory
-#    @logger.should_receive(:info).exactly(15).times.with(/Uploading/)
-#    @logger.should_receive(:info).exactly(1).times.with(/Uploaded/)
-    
-    S3Uploader.upload_directory(@tmp_directory, 'mybucket',
-      { :destination_dir => 'test1/',
-        :logger => @logger,
-        :connection => @connection
-      })
-    
-  end
-    
-end
 
-def create_test_files(directory, number_of_files)
-  FileUtils.mkdir_p directory
+  it 'should upload all files in a directory' do
+    connection = double(:connection)
+    connection.stub_chain(:directories, :new).and_return(directory = double(:directory))
+    directory.stub(:files).and_return(files = double(:files))
+
+    files.should_receive(:create).exactly(12).times
+
+    S3Uploader.upload_directory(tmp_directory, 'mybucket',
+                                { destination_dir: 'test1/',
+                                  logger:          logger,
+                                  connection:      connection })
+  end
+
+  describe 'regexp' do
+
+    it 'should upload specific files' do
+      connection = double(:connection)
+      connection.stub_chain(:directories, :new).and_return(directory = double(:directory))
+      directory.stub(:files).and_return(files = double(:files))
+
+      keys = access.dup
+      files.should_receive(:create).exactly(6).times do |hash|
+        expect(keys).to include(hash[:key])
+        keys.delete(hash[:key])
+      end
+
+      S3Uploader.upload_directory(tmp_directory, 'mybucket',
+                                  { logger:     logger,
+                                    connection: connection,
+                                    regexp:     /access/ })
+    end
+
+  end
+
+  describe 'gzip' do
+
+    it 'should upload compressed files' do
+      connection = double(:connection)
+      connection.stub_chain(:directories, :new).and_return(directory = double(:directory))
+      directory.stub(:files).and_return(files = double(:files))
+
+      #expect to upload gz-files only
+      keys = error.map { |f| f.sub('.gz', '') }.map { |f| f + '.gz' }
+      files.should_receive(:create).exactly(6).times do |hash|
+        expect(keys).to include(hash[:key])
+        keys.delete(hash[:key])
+      end
+
+      S3Uploader.upload_directory(tmp_directory, 'mybucket',
+                                  { logger:     logger,
+                                    connection: connection,
+                                    regexp:     /error/,
+                                    gzip:       true })
+    end
+
+    it 'should use gzip_working_dir correctly' do
+      working_dir = File.join(Dir.tmpdir, 's3uploader_spec/working_dir')
+      FileUtils.mkdir_p working_dir
+      FileUtils.rm_rf(Dir.glob("#{working_dir}/*"))
+
+      connection = double(:connection)
+      connection.stub_chain(:directories, :new).and_return(directory = double(:directory))
+      directory.stub(:files).and_return(files = double(:files))
+
+      #expect to upload gz-files only
+      keys = error.map { |f| f.sub('.gz', '') }.map { |f| f + '.gz' }
+      files.should_receive(:create).exactly(6).times do |hash|
+        expect(keys).to include(hash[:key])
+        keys.delete(hash[:key])
+      end
+
+      S3Uploader.upload_directory(tmp_directory, 'mybucket',
+                                  { logger:           logger,
+                                    connection:       connection,
+                                    regexp:           /error/,
+                                    gzip:             true,
+                                    gzip_working_dir: working_dir })
+
+      #only compress files which aren't compressed yet
+      compressed_files = error.select { |f| File.extname(f) != '.gz' }.map { |f| f + '.gz' }
+      working_dir_content = Dir["#{working_dir}/**/*"].map { |f| f.sub(working_dir, '')[1..-1] }
+
+      #expect compressed files within working_directory
+      expect(working_dir_content & compressed_files).to match_array(compressed_files)
+    end
+
+    it 'when called with bad gzip_working_dir it should raise an exception' do
+      expect {
+        S3Uploader.upload_directory(tmp_directory, 'mybucket',
+                                    { gzip:             true,
+                                      gzip_working_dir: File.join(Dir.tmpdir, 'test_s3_uploader/working_dir') })
+      }.to raise_error('gzip_working_dir may not be located within source-folder')
+
+      expect {
+        S3Uploader.upload_directory(tmp_directory, 'mybucket',
+                                    { gzip:             true,
+                                      gzip_working_dir: File.join(Dir.tmpdir, 'test_s3_uploader_working_dir') })
+      }.to raise_error('gzip_working_dir may not be located within source-folder')
+    end
+
+  end
+
+  describe 'time_range' do
+
+    it 'should not upload any files' do
+      connection = double(:connection)
+      connection.stub_chain(:directories, :new).and_return(directory = double(:directory))
+      directory.stub(:files).and_return(files = double(:files))
+
+      file_names = access.map { |f| "#{tmp_directory}/#{f}" }
+      yesterday  = Time.now - (60 * 60 * 24)
+      File.utime(yesterday, yesterday, *file_names)
+
+      files.should_not_receive(:create)
+
+      S3Uploader.upload_directory(tmp_directory, 'mybucket',
+                                  { logger:     logger,
+                                    connection: connection,
+                                    regexp:     /access/,
+                                    time_range: (Time.now - (60 * 60 * 12))..Time.now })
+    end
+
+    it 'should upload files' do
+      connection = double(:connection)
+      connection.stub_chain(:directories, :new).and_return(directory = double(:directory))
+      directory.stub(:files).and_return(files = double(:files))
+
+      file_names = access.map { |f| "#{tmp_directory}/#{f}" }
+      yesterday  = Time.now - (60 * 60 * 12)
+      File.utime(yesterday, yesterday, *file_names)
+
+      keys = access.dup
+      files.should_receive(:create).exactly(6).times do |hash|
+        expect(keys).to include(hash[:key])
+        keys.delete(hash[:key])
+      end
+
+      S3Uploader.upload_directory(tmp_directory, 'mybucket',
+                                  { logger:     logger,
+                                    connection: connection,
+                                    regexp:     /access/,
+                                    time_range: (Time.now - (60 * 60 * 24))..Time.now })
+    end
+
+  end
 
-  number_of_files.times do |i|
-    Open3.popen3("dd if=/dev/zero of=#{directory}/file#{i}.txt count=1024 bs=1024")
-  end  
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: s3_uploader
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.1.0
 platform: ruby
 authors:
 - Christian Hein
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-29 00:00:00.000000000 Z
+date: 2014-04-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fog
@@ -28,16 +28,16 @@ dependencies:
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.14.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.14.1
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -92,7 +92,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.1
+rubygems_version: 2.1.2
 signing_key: 
 specification_version: 4
 summary: S3 multithreaded directory uploader