s3_uploader 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/.gitignore ADDED
@@ -0,0 +1,17 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in s3_uploader.gemspec
4
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2012 Christian Hein
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,39 @@
1
+ # S3Uploader
2
+
3
+ Multithreaded recursive directory uploader to S3 using Fog
4
+
5
+ ## Installation
6
+
7
+ Add this line to your application's Gemfile:
8
+
9
+ gem 's3_uploader'
10
+
11
+ And then execute:
12
+
13
+ $ bundle
14
+
15
+ Or install it yourself as:
16
+
17
+ $ gem install s3_uploader
18
+
19
+ ## Usage
20
+
21
+ S3Uploader.upload_directory('/tmp/test', 'mybucket',
22
+ { :s3_key => YOUR_KEY, :s3_secret => YOUR_SECRET_KEY,
23
+ :destination_dir => 'test/', :threads => 4 })
24
+
25
+ If no keys are provided, it uses S3_KEY and S3_SECRET environment variables.
26
+
27
+ S3Uploader.upload_directory('/tmp/test', 'mybucket', { :destination_dir => 'test/', :threads => 4 })
28
+
29
+ Or as a command line binary
30
+
31
+ s3uploader -d test/ -t 4 /tmp/test mybucket
32
+
33
+ ## Contributing
34
+
35
+ 1. Fork it
36
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
37
+ 3. Commit your changes (`git commit -am 'Added some feature'`)
38
+ 4. Push to the branch (`git push origin my-new-feature`)
39
+ 5. Create new Pull Request
data/Rakefile ADDED
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env rake
2
+ require "bundler/gem_tasks"
3
+ require 'rspec/core/rake_task'
4
+
5
+ RSpec::Core::RakeTask.new('spec')
data/bin/s3uploader ADDED
@@ -0,0 +1,48 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 's3_uploader'
4
+ require 'optparse'
5
+
6
+ options = {}
7
+ source = nil
8
+ bucket = nil
9
+
10
+ opt_parser = OptionParser.new do |opt|
11
+ opt.banner = "Usage: s3uploader [OPTIONS] SOURCE BUCKET"
12
+ opt.separator ""
13
+ opt.separator "Options"
14
+
15
+ opt.on("-t","--threads THREADS","number of concurrent threads") do |threads|
16
+ options[:threads] = threads.to_i
17
+ end
18
+
19
+ opt.on("-d","--destination DESTINATION_DIR","destination directory") do |destination_dir|
20
+ options[:destination_dir] = destination_dir
21
+ end
22
+
23
+ opt.on("-p","--public PUBLIC","set ACL of files to public. Default: false") do |public_files|
24
+ options[:public] = public_files
25
+ end
26
+
27
+ opt.on("-k","--key KEY","S3 key. Default: environment variable S3_KEY") do |s3_key|
28
+ options[:s3_key] = s3_key
29
+ end
30
+
31
+ opt.on("-s","--secret SECRET","S3 secret key. Default: environment variable S3_SECRET") do |s3_secret|
32
+ options[:s3_secret] = s3_secret
33
+ end
34
+
35
+ opt.on("-h","--help","help") do
36
+ puts opt_parser
37
+ exit
38
+ end
39
+ end
40
+
41
+ opt_parser.parse!
42
+
43
+ if !ARGV[0] or !ARGV[1]
44
+ puts "Missing source directory or destination bucket"
45
+ exit
46
+ end
47
+
48
+ S3Uploader.upload_directory(ARGV[0], ARGV[1], options)
@@ -0,0 +1,4 @@
1
+ require 'fog'
2
+ require 'thread'
3
+ require "s3_uploader/version"
4
+ require "s3_uploader/s3_uploader"
@@ -0,0 +1,59 @@
1
+ module S3Uploader
2
+ def self.upload_directory(source, bucket, options = {})
3
+ options = {
4
+ :destination_dir => '',
5
+ :threads => 5,
6
+ :s3_key => ENV['S3_KEY'],
7
+ :s3_secret => ENV['S3_SECRET'],
8
+ :public => false
9
+ }.merge(options)
10
+
11
+ raise 'Source must be a directory' unless File.directory?(source)
12
+
13
+ source.chop! if source.end_with?('/')
14
+ if options[:destination_dir] != '' and !options[:destination_dir].end_with?('/')
15
+ options[:destination_dir] = "#{options[:destination_dir]}/"
16
+ end
17
+
18
+ files = Queue.new
19
+ Dir.glob("#{source}/**/*").select{ |f| !File.directory?(f) }.each do |f|
20
+ files << f
21
+ end
22
+
23
+ connection = Fog::Storage.new({
24
+ :provider => 'AWS',
25
+ :aws_access_key_id => options[:s3_key],
26
+ :aws_secret_access_key => options[:s3_secret]
27
+ })
28
+
29
+ directory = connection.directories.new(:key => bucket)
30
+
31
+ total_files = files.size
32
+ file_number = 0
33
+ @mutex = Mutex.new
34
+
35
+ threads = []
36
+ options[:threads].times do |i|
37
+ threads[i] = Thread.new {
38
+
39
+ while not files.empty?
40
+ @mutex.synchronize do
41
+ file_number += 1
42
+ end
43
+ file = files.pop
44
+ key = file.gsub(source, '')[1..-1]
45
+ dest = "#{options[:destination_dir]}#{key}"
46
+ puts "[#{file_number}/#{total_files}] Uploading #{key} to s3://#{bucket}/#{dest}"
47
+
48
+ directory.files.create(
49
+ :key => dest,
50
+ :body => File.open(file),
51
+ :public => options[:public]
52
+ )
53
+ end
54
+ }
55
+ end
56
+ threads.each { |t| t.join }
57
+
58
+ end
59
+ end
@@ -0,0 +1,3 @@
1
+ module S3Uploader
2
+ VERSION = "0.0.1"
3
+ end
@@ -0,0 +1,22 @@
1
+ # -*- encoding: utf-8 -*-
2
+ require File.expand_path('../lib/s3_uploader/version', __FILE__)
3
+
4
+ Gem::Specification.new do |gem|
5
+ gem.authors = ["Christian Hein"]
6
+ gem.email = ["chrishein@gmail.com"]
7
+ gem.description = %q{S3 multithreaded directory uploader}
8
+ gem.summary = %q{S3 multithreaded directory uploader}
9
+ gem.homepage = ""
10
+
11
+ gem.files = `git ls-files`.split($\)
12
+ gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
13
+ gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
14
+ gem.name = "s3_uploader"
15
+ gem.require_paths = ["lib"]
16
+ gem.version = S3Uploader::VERSION
17
+
18
+ gem.add_dependency 'fog'
19
+
20
+ gem.add_development_dependency 'rspec'
21
+ gem.add_development_dependency 'rake'
22
+ end
@@ -0,0 +1,9 @@
1
+ require 'spec_helper'
2
+
3
+ describe S3Uploader do
4
+
5
+ it "should upload all files in a directory" do
6
+ pending 'Write real test'
7
+ S3Uploader.upload_directory('/tmp/test', 'mybucket', { :destination_dir => 'test/' })
8
+ end
9
+ end
@@ -0,0 +1,7 @@
1
+ require 'rspec'
2
+ require 's3_uploader'
3
+
4
+ RSpec.configure do |config|
5
+ config.color_enabled = true
6
+ config.formatter = 'documentation'
7
+ end
metadata ADDED
@@ -0,0 +1,93 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: s3_uploader
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - Christian Hein
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2012-09-14 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: fog
16
+ requirement: &2152628560 !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ! '>='
20
+ - !ruby/object:Gem::Version
21
+ version: '0'
22
+ type: :runtime
23
+ prerelease: false
24
+ version_requirements: *2152628560
25
+ - !ruby/object:Gem::Dependency
26
+ name: rspec
27
+ requirement: &2152627500 !ruby/object:Gem::Requirement
28
+ none: false
29
+ requirements:
30
+ - - ! '>='
31
+ - !ruby/object:Gem::Version
32
+ version: '0'
33
+ type: :development
34
+ prerelease: false
35
+ version_requirements: *2152627500
36
+ - !ruby/object:Gem::Dependency
37
+ name: rake
38
+ requirement: &2152626500 !ruby/object:Gem::Requirement
39
+ none: false
40
+ requirements:
41
+ - - ! '>='
42
+ - !ruby/object:Gem::Version
43
+ version: '0'
44
+ type: :development
45
+ prerelease: false
46
+ version_requirements: *2152626500
47
+ description: S3 multithreaded directory uploader
48
+ email:
49
+ - chrishein@gmail.com
50
+ executables:
51
+ - s3uploader
52
+ extensions: []
53
+ extra_rdoc_files: []
54
+ files:
55
+ - .gitignore
56
+ - Gemfile
57
+ - LICENSE
58
+ - README.md
59
+ - Rakefile
60
+ - bin/s3uploader
61
+ - lib/s3_uploader.rb
62
+ - lib/s3_uploader/s3_uploader.rb
63
+ - lib/s3_uploader/version.rb
64
+ - s3_uploader.gemspec
65
+ - spec/s3uploader_spec.rb
66
+ - spec/spec_helper.rb
67
+ homepage: ''
68
+ licenses: []
69
+ post_install_message:
70
+ rdoc_options: []
71
+ require_paths:
72
+ - lib
73
+ required_ruby_version: !ruby/object:Gem::Requirement
74
+ none: false
75
+ requirements:
76
+ - - ! '>='
77
+ - !ruby/object:Gem::Version
78
+ version: '0'
79
+ required_rubygems_version: !ruby/object:Gem::Requirement
80
+ none: false
81
+ requirements:
82
+ - - ! '>='
83
+ - !ruby/object:Gem::Version
84
+ version: '0'
85
+ requirements: []
86
+ rubyforge_project:
87
+ rubygems_version: 1.8.17
88
+ signing_key:
89
+ specification_version: 3
90
+ summary: S3 multithreaded directory uploader
91
+ test_files:
92
+ - spec/s3uploader_spec.rb
93
+ - spec/spec_helper.rb