s3_uploader 0.0.5 → 0.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,15 @@
1
+ ---
2
+ !binary "U0hBMQ==":
3
+ metadata.gz: !binary |-
4
+ ZDRkNjdiMDIyYmU3ZWE0NDc3OTNlYjU3MTU3NzM5NjhiYTFkYTcxOA==
5
+ data.tar.gz: !binary |-
6
+ M2YyODEyNjliNDc4YTkwNDcwYmIzZTdhN2QyYTFkYWU0MDM2MGE1Mw==
7
+ SHA512:
8
+ metadata.gz: !binary |-
9
+ MGQyMDhmZWEyNzMwYzQxMzhkYmYwYWM0NWZhYzY1ZTQ1MmQ0ZTg2M2VlMDFh
10
+ NjVlYzg5NDc2ZDFiYjk1NGFjYzY5NjcxYTE4YmFjODc0MmRhYjcwYzQ1NGJh
11
+ MWQwZmE4YjMyNDU2MDA0MmI2ZWM0YjI4ZGY5OGVjZjM5Y2ZjNmE=
12
+ data.tar.gz: !binary |-
13
+ ZTU0ZGRhNTU4NDg1MDAwOWZlNDFiZDU1YWQ3NmYxMDI4MDlmYmExN2IxZDNl
14
+ YmU5NjNhOThmYmZiNzVmYzdjYWYxY2VhNjQ0Mzk4N2RjNjE0M2E2MTlkNGEz
15
+ N2JlMGIxZGYxMjgxOTU3MDEwOWM3OGMxMmNkNzhiM2JkOGY5NmU=
data/README.md CHANGED
@@ -48,9 +48,8 @@ Again, it uses S3_KEY and S3_SECRET environment variables if non provided in par
48
48
 
49
49
  ## TODO
50
50
 
51
- 1. Write test suite
52
- 2. Allow regex pattern matching to select files to upload
53
- 3. Add optional time, size and number of files uploaded report at end of process
51
+ 1. Allow regex pattern matching to select files to upload
52
+ 2. Add optional time, size and number of files uploaded report at end of process
54
53
 
55
54
  ## Contributing
56
55
 
@@ -1,4 +1,5 @@
1
1
  module S3Uploader
2
+ KILO_SIZE = 1024.0
2
3
  def self.upload_directory(source, bucket, options = {})
3
4
  options = {
4
5
  :destination_dir => '',
@@ -13,19 +14,11 @@ module S3Uploader
13
14
 
14
15
  raise 'Source must be a directory' unless File.directory?(source)
15
16
 
16
- source = source.chop if source.end_with?('/')
17
- if options[:destination_dir] != '' and !options[:destination_dir].end_with?('/')
18
- options[:destination_dir] = "#{options[:destination_dir]}/"
19
- end
20
-
21
- files = Queue.new
22
- Dir.glob("#{source}/**/*").select{ |f| !File.directory?(f) }.each do |f|
23
- files << f
24
- end
25
-
26
17
  if options[:connection]
27
18
  connection = options[:connection]
28
19
  else
20
+ raise "Missing access keys" if options[:s3_key].nil? or options[:s3_secret].nil?
21
+
29
22
  connection = Fog::Storage.new({
30
23
  :provider => 'AWS',
31
24
  :aws_access_key_id => options[:s3_key],
@@ -34,8 +27,21 @@ module S3Uploader
34
27
  })
35
28
  end
36
29
 
30
+ source = source.chop if source.end_with?('/')
31
+ if options[:destination_dir] != '' and !options[:destination_dir].end_with?('/')
32
+ options[:destination_dir] = "#{options[:destination_dir]}/"
33
+ end
34
+ total_size = 0
35
+ files = Queue.new
36
+ Dir.glob("#{source}/**/*").select{ |f| !File.directory?(f) }.each do |f|
37
+ files << f
38
+ total_size += File.size(f)
39
+
40
+ end
41
+
37
42
  directory = connection.directories.new(:key => bucket)
38
43
 
44
+ start = Time.now
39
45
  total_files = files.size
40
46
  file_number = 0
41
47
  @mutex = Mutex.new
@@ -44,24 +50,32 @@ module S3Uploader
44
50
  options[:threads].times do |i|
45
51
  threads[i] = Thread.new {
46
52
 
47
- while not files.empty?
53
+ until files.empty?
48
54
  @mutex.synchronize do
49
55
  file_number += 1
56
+ Thread.current["file_number"] = file_number
50
57
  end
51
- file = files.pop
52
- key = file.gsub(source, '')[1..-1]
53
- dest = "#{options[:destination_dir]}#{key}"
54
- log.info("[#{file_number}/#{total_files}] Uploading #{key} to s3://#{bucket}/#{dest}")
58
+ file = files.pop rescue nil
59
+ if file
60
+ key = file.gsub(source, '')[1..-1]
61
+ dest = "#{options[:destination_dir]}#{key}"
62
+ log.info("[#{Thread.current["file_number"]}/#{total_files}] Uploading #{key} to s3://#{bucket}/#{dest}")
55
63
 
56
- directory.files.create(
57
- :key => dest,
58
- :body => File.open(file),
59
- :public => options[:public]
60
- )
64
+ directory.files.create(
65
+ :key => dest,
66
+ :body => File.open(file),
67
+ :public => options[:public]
68
+ )
69
+ end
61
70
  end
62
71
  }
63
72
  end
64
73
  threads.each { |t| t.join }
65
74
 
75
+ finish = Time.now
76
+ elapsed = finish.to_f - start.to_f
77
+ mins, secs = elapsed.divmod 60.0
78
+ log.info("Uploaded %d (%.#{0}f KB) in %d:%04.2f" % [total_files, total_size / KILO_SIZE, mins.to_i, secs])
79
+
66
80
  end
67
81
  end
@@ -1,3 +1,3 @@
1
1
  module S3Uploader
2
- VERSION = "0.0.5"
2
+ VERSION = "0.0.6"
3
3
  end
@@ -1,37 +1,56 @@
1
1
  require 'spec_helper'
2
2
 
3
- describe S3Uploader do
4
- before :all do
5
- Fog.mock!
6
-
7
- @connection = Fog::Storage.new({
8
- :provider => 'AWS',
9
- :aws_access_key_id => '11111111111',
10
- :aws_secret_access_key => 'XXXXXXXXXXXXXXXXXXXXXXXXXXX'
11
- })
12
-
13
- @connection.directories.create(
14
- :key => 'mybucket',
15
- :public => true
16
- )
17
-
18
- @tmp_directory = File.join(Dir.tmpdir, 'test_s3_uploader')
19
- create_test_files(@tmp_directory, 10)
20
- create_test_files(File.join(@tmp_directory, 'subdir1'), 5)
21
- @logger = Logger.new(StringIO.new)
22
- end
23
-
24
- it "should upload all files in a directory" do
25
-
26
- @logger.should_receive(:info).exactly(15).times.with(/Uploading/)
27
-
28
- S3Uploader.upload_directory(@tmp_directory, 'mybucket',
3
+ describe S3Uploader do
4
+
5
+ it 'when called with missing access keys it should raise an exception' do
6
+ lambda {
7
+ S3Uploader.upload_directory('/tmp', 'mybucket',
29
8
  { :destination_dir => 'test1/',
30
- :logger => @logger,
31
- :connection => @connection
9
+ :s3_key => nil,
10
+ :s3_secret => nil
32
11
  })
33
-
34
- end
12
+ }.should raise_error('Missing access keys')
13
+ end
14
+
15
+ it 'when called with source not directory it should raise an exception' do
16
+ lambda {
17
+ S3Uploader.upload_directory('/xzzaz1232', 'mybucket')
18
+ }.should raise_error('Source must be a directory')
19
+ end
20
+
21
+
22
+ before :all do
23
+ Fog.mock!
24
+
25
+ @connection = Fog::Storage.new({
26
+ :provider => 'AWS',
27
+ :aws_access_key_id => '11111111111',
28
+ :aws_secret_access_key => 'XXXXXXXXXXXXXXXXXXXXXXXXXXX'
29
+ })
30
+
31
+ @connection.directories.create(
32
+ :key => 'mybucket',
33
+ :public => true
34
+ )
35
+
36
+ @tmp_directory = File.join(Dir.tmpdir, 'test_s3_uploader')
37
+ create_test_files(@tmp_directory, 10)
38
+ create_test_files(File.join(@tmp_directory, 'subdir1'), 5)
39
+ @logger = Logger.new(STDOUT)
40
+ end
41
+
42
+ it "should upload all files in a directory" do
43
+ puts @tmp_directory
44
+ # @logger.should_receive(:info).exactly(15).times.with(/Uploading/)
45
+ # @logger.should_receive(:info).exactly(1).times.with(/Uploaded/)
46
+
47
+ S3Uploader.upload_directory(@tmp_directory, 'mybucket',
48
+ { :destination_dir => 'test1/',
49
+ :logger => @logger,
50
+ :connection => @connection
51
+ })
52
+
53
+ end
35
54
 
36
55
  end
37
56
 
@@ -39,6 +58,6 @@ def create_test_files(directory, number_of_files)
39
58
  FileUtils.mkdir_p directory
40
59
 
41
60
  number_of_files.times do |i|
42
- Open3.popen3('dd if=/dev/zero of=#{directory}/file#{i}.txt count=1024 bs=1024')
61
+ Open3.popen3("dd if=/dev/zero of=#{directory}/file#{i}.txt count=1024 bs=1024")
43
62
  end
44
63
  end
metadata CHANGED
@@ -1,49 +1,57 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: s3_uploader
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.5
5
- prerelease:
4
+ version: 0.0.6
6
5
  platform: ruby
7
6
  authors:
8
7
  - Christian Hein
9
8
  autorequire:
10
9
  bindir: bin
11
10
  cert_chain: []
12
- date: 2012-09-22 00:00:00.000000000 Z
11
+ date: 2013-09-29 00:00:00.000000000 Z
13
12
  dependencies:
14
13
  - !ruby/object:Gem::Dependency
15
14
  name: fog
16
- requirement: &2152669100 !ruby/object:Gem::Requirement
17
- none: false
15
+ requirement: !ruby/object:Gem::Requirement
18
16
  requirements:
19
17
  - - ! '>='
20
18
  - !ruby/object:Gem::Version
21
19
  version: '0'
22
20
  type: :runtime
23
21
  prerelease: false
24
- version_requirements: *2152669100
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ! '>='
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
25
27
  - !ruby/object:Gem::Dependency
26
28
  name: rspec
27
- requirement: &2152668680 !ruby/object:Gem::Requirement
28
- none: false
29
+ requirement: !ruby/object:Gem::Requirement
29
30
  requirements:
30
31
  - - ! '>='
31
32
  - !ruby/object:Gem::Version
32
33
  version: '0'
33
34
  type: :development
34
35
  prerelease: false
35
- version_requirements: *2152668680
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ! '>='
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
36
41
  - !ruby/object:Gem::Dependency
37
42
  name: rake
38
- requirement: &2152668260 !ruby/object:Gem::Requirement
39
- none: false
43
+ requirement: !ruby/object:Gem::Requirement
40
44
  requirements:
41
45
  - - ! '>='
42
46
  - !ruby/object:Gem::Version
43
47
  version: '0'
44
48
  type: :development
45
49
  prerelease: false
46
- version_requirements: *2152668260
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ! '>='
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
47
55
  description: S3 multithreaded directory uploader
48
56
  email:
49
57
  - chrishein@gmail.com
@@ -66,28 +74,28 @@ files:
66
74
  - spec/spec_helper.rb
67
75
  homepage: ''
68
76
  licenses: []
77
+ metadata: {}
69
78
  post_install_message:
70
79
  rdoc_options: []
71
80
  require_paths:
72
81
  - lib
73
82
  required_ruby_version: !ruby/object:Gem::Requirement
74
- none: false
75
83
  requirements:
76
84
  - - ! '>='
77
85
  - !ruby/object:Gem::Version
78
86
  version: '0'
79
87
  required_rubygems_version: !ruby/object:Gem::Requirement
80
- none: false
81
88
  requirements:
82
89
  - - ! '>='
83
90
  - !ruby/object:Gem::Version
84
91
  version: '0'
85
92
  requirements: []
86
93
  rubyforge_project:
87
- rubygems_version: 1.8.17
94
+ rubygems_version: 2.1.2
88
95
  signing_key:
89
- specification_version: 3
96
+ specification_version: 4
90
97
  summary: S3 multithreaded directory uploader
91
98
  test_files:
92
99
  - spec/s3uploader_spec.rb
93
100
  - spec/spec_helper.rb
101
+ has_rdoc: