s3_uploader 0.0.5 → 0.0.6

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ZDRkNjdiMDIyYmU3ZWE0NDc3OTNlYjU3MTU3NzM5NjhiYTFkYTcxOA==
+  data.tar.gz: !binary |-
+    M2YyODEyNjliNDc4YTkwNDcwYmIzZTdhN2QyYTFkYWU0MDM2MGE1Mw==
+SHA512:
+  metadata.gz: !binary |-
+    MGQyMDhmZWEyNzMwYzQxMzhkYmYwYWM0NWZhYzY1ZTQ1MmQ0ZTg2M2VlMDFh
+    NjVlYzg5NDc2ZDFiYjk1NGFjYzY5NjcxYTE4YmFjODc0MmRhYjcwYzQ1NGJh
+    MWQwZmE4YjMyNDU2MDA0MmI2ZWM0YjI4ZGY5OGVjZjM5Y2ZjNmE=
+  data.tar.gz: !binary |-
+    ZTU0ZGRhNTU4NDg1MDAwOWZlNDFiZDU1YWQ3NmYxMDI4MDlmYmExN2IxZDNl
+    YmU5NjNhOThmYmZiNzVmYzdjYWYxY2VhNjQ0Mzk4N2RjNjE0M2E2MTlkNGEz
+    N2JlMGIxZGYxMjgxOTU3MDEwOWM3OGMxMmNkNzhiM2JkOGY5NmU=

data/README.md

@@ -48,9 +48,8 @@ Again, it uses S3_KEY and S3_SECRET environment variables if non provided in par
 
 ## TODO
 
-1. Write test suite
-2. Allow regex pattern matching to select files to upload
-3. Add optional time, size and number of files uploaded report at end of process
+1. Allow regex pattern matching to select files to upload
+2. Add optional time, size and number of files uploaded report at end of process
 
 ## Contributing
 

data/lib/s3_uploader/s3_uploader.rb

@@ -1,4 +1,5 @@
 module S3Uploader
+  KILO_SIZE = 1024.0
   def self.upload_directory(source, bucket, options = {})
     options = {
       :destination_dir => '',
@@ -13,19 +14,11 @@ module S3Uploader
     
     raise 'Source must be a directory' unless File.directory?(source)
     
-    source = source.chop if source.end_with?('/')
-    if options[:destination_dir] != '' and !options[:destination_dir].end_with?('/')
-      options[:destination_dir] = "#{options[:destination_dir]}/"
-    end
-    
-    files = Queue.new
-    Dir.glob("#{source}/**/*").select{ |f| !File.directory?(f) }.each do |f|
-      files << f
-    end
-    
     if options[:connection]
       connection = options[:connection]
     else
+      raise "Missing access keys" if options[:s3_key].nil? or options[:s3_secret].nil?
+      
       connection = Fog::Storage.new({
           :provider => 'AWS',
           :aws_access_key_id => options[:s3_key],
@@ -34,8 +27,21 @@ module S3Uploader
       })
     end
     
+    source = source.chop if source.end_with?('/')
+    if options[:destination_dir] != '' and !options[:destination_dir].end_with?('/')
+      options[:destination_dir] = "#{options[:destination_dir]}/"
+    end
+    total_size = 0
+    files = Queue.new
+    Dir.glob("#{source}/**/*").select{ |f| !File.directory?(f) }.each do |f|
+      files << f
+      total_size += File.size(f)
+      
+    end
+    
     directory = connection.directories.new(:key => bucket)
     
+    start = Time.now
     total_files = files.size
     file_number = 0
     @mutex = Mutex.new
@@ -44,24 +50,32 @@ module S3Uploader
     options[:threads].times do |i|
       threads[i] = Thread.new {
         
-        while not files.empty?
+        until files.empty?
           @mutex.synchronize do
             file_number += 1
+            Thread.current["file_number"] = file_number
           end
-          file = files.pop
-          key = file.gsub(source, '')[1..-1]
-          dest = "#{options[:destination_dir]}#{key}"
-          log.info("[#{file_number}/#{total_files}] Uploading #{key} to s3://#{bucket}/#{dest}")
+          file = files.pop rescue nil
+          if file
+            key = file.gsub(source, '')[1..-1]
+            dest = "#{options[:destination_dir]}#{key}"
+            log.info("[#{Thread.current["file_number"]}/#{total_files}] Uploading #{key} to s3://#{bucket}/#{dest}")
           
-          directory.files.create(
-            :key    => dest,
-            :body   => File.open(file),
-            :public => options[:public]
-          )
+            directory.files.create(
+              :key    => dest,
+              :body   => File.open(file),
+              :public => options[:public]
+            )
+          end
         end 
       }
     end
     threads.each { |t| t.join }
     
+    finish = Time.now
+    elapsed = finish.to_f - start.to_f
+    mins, secs = elapsed.divmod 60.0
+    log.info("Uploaded %d (%.#{0}f KB) in %d:%04.2f" % [total_files, total_size / KILO_SIZE, mins.to_i, secs])
+    
   end
 end

data/lib/s3_uploader/version.rb

@@ -1,3 +1,3 @@
 module S3Uploader
-  VERSION = "0.0.5"
+  VERSION = "0.0.6"
 end

data/spec/s3uploader_spec.rb

@@ -1,37 +1,56 @@
 require 'spec_helper'
 
-describe S3Uploader do  
-    before :all do
-      Fog.mock!
-      
-      @connection = Fog::Storage.new({
-        :provider                 => 'AWS',
-        :aws_access_key_id        => '11111111111',
-        :aws_secret_access_key    => 'XXXXXXXXXXXXXXXXXXXXXXXXXXX'
-      })
-      
-      @connection.directories.create(
-        :key    => 'mybucket',
-        :public => true
-      )
-      
-      @tmp_directory = File.join(Dir.tmpdir, 'test_s3_uploader')
-      create_test_files(@tmp_directory, 10)
-      create_test_files(File.join(@tmp_directory, 'subdir1'), 5)
-      @logger = Logger.new(StringIO.new)
-    end
-    
-    it "should upload all files in a directory" do
-      
-      @logger.should_receive(:info).exactly(15).times.with(/Uploading/)
-      
-      S3Uploader.upload_directory(@tmp_directory, 'mybucket',
+describe S3Uploader do
+  
+  it 'when called with missing access keys it should raise an exception' do
+    lambda {
+      S3Uploader.upload_directory('/tmp', 'mybucket',
         { :destination_dir => 'test1/',
-          :logger => @logger,
-          :connection => @connection
+          :s3_key => nil,
+          :s3_secret => nil
         })
-      
-    end
+    }.should raise_error('Missing access keys')
+  end
+  
+  it 'when called with source not directory it should raise an exception' do
+    lambda {
+      S3Uploader.upload_directory('/xzzaz1232', 'mybucket')
+    }.should raise_error('Source must be a directory')
+  end
+  
+  
+  before :all do
+    Fog.mock!
+    
+    @connection = Fog::Storage.new({
+      :provider                 => 'AWS',
+      :aws_access_key_id        => '11111111111',
+      :aws_secret_access_key    => 'XXXXXXXXXXXXXXXXXXXXXXXXXXX'
+    })
+    
+    @connection.directories.create(
+      :key    => 'mybucket',
+      :public => true
+    )
+    
+    @tmp_directory = File.join(Dir.tmpdir, 'test_s3_uploader')
+    create_test_files(@tmp_directory, 10)
+    create_test_files(File.join(@tmp_directory, 'subdir1'), 5)
+    @logger = Logger.new(STDOUT)
+  end
+  
+  it "should upload all files in a directory" do
+    puts @tmp_directory
+#    @logger.should_receive(:info).exactly(15).times.with(/Uploading/)
+#    @logger.should_receive(:info).exactly(1).times.with(/Uploaded/)
+    
+    S3Uploader.upload_directory(@tmp_directory, 'mybucket',
+      { :destination_dir => 'test1/',
+        :logger => @logger,
+        :connection => @connection
+      })
+    
+  end
     
 end
 
@@ -39,6 +58,6 @@ def create_test_files(directory, number_of_files)
   FileUtils.mkdir_p directory
 
   number_of_files.times do |i|
-    Open3.popen3('dd if=/dev/zero of=#{directory}/file#{i}.txt count=1024 bs=1024')
+    Open3.popen3("dd if=/dev/zero of=#{directory}/file#{i}.txt count=1024 bs=1024")
   end  
 end

metadata

@@ -1,49 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: s3_uploader
 version: !ruby/object:Gem::Version
-  version: 0.0.5
-  prerelease: 
+  version: 0.0.6
 platform: ruby
 authors:
 - Christian Hein
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-22 00:00:00.000000000 Z
+date: 2013-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fog
-  requirement: &2152669100 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2152669100
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2152668680 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2152668680
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &2152668260 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2152668260
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: S3 multithreaded directory uploader
 email:
 - chrishein@gmail.com
@@ -66,28 +74,28 @@ files:
 - spec/spec_helper.rb
 homepage: ''
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.17
+rubygems_version: 2.1.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: S3 multithreaded directory uploader
 test_files:
 - spec/s3uploader_spec.rb
 - spec/spec_helper.rb
+has_rdoc: