s3-publisher 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/.document ADDED
@@ -0,0 +1,5 @@
1
+ README.rdoc
2
+ lib/**/*.rb
3
+ bin/*
4
+ features/**/*.feature
5
+ LICENSE
data/.gitignore ADDED
@@ -0,0 +1,5 @@
1
+ *.sw?
2
+ .DS_Store
3
+ coverage
4
+ rdoc
5
+ pkg
data/LICENSE ADDED
@@ -0,0 +1,20 @@
1
+ Copyright (c) 2009 Ben Koski
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.rdoc ADDED
@@ -0,0 +1,38 @@
1
+ = s3-publisher
2
+
3
+ S3Publisher is meant as a clean, simple, sensible-defaults way to publish
4
+ files to Amazon S3 for the world to see.
5
+
6
+ Basic usage:
7
+
8
+ require 's3-publisher'
9
+ S3Publisher.publish('my-bucket') do |p|
10
+ p.push('test.txt', 'abc1234')
11
+ end
12
+
13
+ This will:
14
+ * push test.txt to my-bucket.s3.amazonaws.com
15
+ * use reduced redundancy storage
16
+ * set security to public-read
17
+ * gzip contents ('abc1234') and set a Content-Encoding: gzip header so clients know to decompress
18
+ * set a Cache-Control: max-age=5 header
19
+
20
+ Slightly more advanced example:
21
+
22
+ S3Publisher.publish('my-bucket', :base_path => 'world_cup') do |p|
23
+ p.push('events.xml', '<xml>...', :ttl => 15)
24
+ end
25
+
26
+ In this example:
27
+ * file will be written to my-bucket.s3.amazonaws.com/world_cup/events.xml
28
+ * Cache-Control: max-age=15 will be set
29
+
30
+ A few miscellaneous notes:
31
+ * gzip compress is skipped on .jpg/gif/png/tif files
32
+ * uploads are multi-threaded. You can control worker thread count on instantiation.
33
+
34
+ See class docs for further options.
35
+
36
+ == Copyright
37
+
38
+ Copyright (c) 2010 Ben Koski. See LICENSE for details.
data/Rakefile ADDED
@@ -0,0 +1,58 @@
1
+ require 'rubygems'
2
+ require 'rake'
3
+
4
+ begin
5
+ require 'jeweler'
6
+ Jeweler::Tasks.new do |gem|
7
+ gem.name = "s3-publisher"
8
+ gem.summary = %Q{Publish data to S3 for the world to see}
9
+ gem.description = %Q{Publish data to S3 for the world to see}
10
+ gem.email = "gems@benkoski.com"
11
+ gem.homepage = "http://github.com/bkoski/s3-publisher"
12
+ gem.authors = ["Ben Koski"]
13
+ gem.add_development_dependency "thoughtbot-shoulda"
14
+ gem.add_dependency 'aws_credentials'
15
+ gem.add_dependency 'right_aws', '=2.0.0'
16
+ # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
17
+ end
18
+ rescue LoadError
19
+ puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
20
+ end
21
+
22
+ require 'rake/testtask'
23
+ Rake::TestTask.new(:test) do |test|
24
+ test.libs << 'lib' << 'test'
25
+ test.pattern = 'test/**/*_test.rb'
26
+ test.verbose = true
27
+ end
28
+
29
+ begin
30
+ require 'rcov/rcovtask'
31
+ Rcov::RcovTask.new do |test|
32
+ test.libs << 'test'
33
+ test.pattern = 'test/**/*_test.rb'
34
+ test.verbose = true
35
+ end
36
+ rescue LoadError
37
+ task :rcov do
38
+ abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
39
+ end
40
+ end
41
+
42
+ task :test => :check_dependencies
43
+
44
+ task :default => :test
45
+
46
+ require 'rake/rdoctask'
47
+ Rake::RDocTask.new do |rdoc|
48
+ if File.exist?('VERSION')
49
+ version = File.read('VERSION')
50
+ else
51
+ version = ""
52
+ end
53
+
54
+ rdoc.rdoc_dir = 'rdoc'
55
+ rdoc.title = "s3-publisher #{version}"
56
+ rdoc.rdoc_files.include('README*')
57
+ rdoc.rdoc_files.include('lib/**/*.rb')
58
+ end
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 0.4.0
@@ -0,0 +1,128 @@
1
+ require 'rubygems'
2
+
3
+ gem 'right_aws', '=2.0.0'
4
+ require 'right_aws'
5
+
6
+ require 'aws_credentials'
7
+ require 'zlib'
8
+ require 'thread'
9
+
10
+ Thread.abort_on_exception = true
11
+
12
+ # You can either use the block syntax, or:
13
+ # * instantiate a class
14
+ # * queue data to be published with push
15
+ # * call run to actually upload the data to S3
16
+ class S3Publisher
17
+
18
+ attr_reader :bucket_name, :base_path, :logger, :workers_to_use
19
+
20
+ # Block style. run is called for you on block close.
21
+ # S3Publisher.publish('my-bucket') do |p|
22
+ # p.push('test.txt', '123abc')
23
+ # end
24
+ def self.publish bucket_name, opts={}, &block
25
+ p = self.new(bucket_name, opts)
26
+ yield(p)
27
+ p.run
28
+ end
29
+
30
+ # Pass the publisher a bucket_name along with any of the following options:
31
+ # * <tt>base_path</tt> - path prepended to supplied file_name on upload
32
+ # * <tt>logger</tt> - a logger object to recieve 'uploaded' messages. Defaults to STDOUT.
33
+ # * <tt>workers</tt> - number of threads to use when pushing to S3. Defaults to 3.
34
+ def initialize bucket_name, opts={}
35
+ @s3 = RightAws::S3.new(AWSCredentials.access_key, AWSCredentials.secret_access_key, :multi_thread => true,
36
+ :protocol => 'http',
37
+ :port => 80,
38
+ :logger => Logger.new(nil))
39
+ @bucket_name, @base_path = bucket_name, opts[:base_path]
40
+ @logger = opts[:logger] || STDOUT
41
+ @workers_to_use = opts[:workers] || 3
42
+ @publish_queue = Queue.new
43
+ end
44
+
45
+ # Pass:
46
+ # * <tt>file_name</tt> - name of file on S3. base_path will be prepended if supplied on instantiate.
47
+ # * <tt>data</tt> - data to be uploaded as a string
48
+ #
49
+ # And one or many options:
50
+ # * <tt>:gzip (true|false)</tt> - gzip file contents? defaults to true.
51
+ # * <tt>:ttl</tt> - TTL in seconds for cache-control header. defaults to 5.
52
+ # * <tt>:cache_control</tt> - specify Cache-Control header directly if you don't like the default
53
+ # * <tt>:content_type</tt> - no need to specify if default based on extension is okay. But if you need to force,
54
+ # you can provide :xml, :html, :text, or your own custom string.
55
+ # * <tt>:redundancy</tt> - by default objects are stored at reduced redundancy, pass :standard to store at full
56
+ def push file_name, data, opts={}
57
+ headers = {}
58
+
59
+ file_name = "#{base_path}/#{file_name}" unless base_path.nil?
60
+
61
+ unless opts[:gzip] == false || file_name.match(/\.(jpg|gif|png|tif)$/)
62
+ data = gzip(data)
63
+ headers['Content-Encoding'] = 'gzip'
64
+ end
65
+
66
+ headers['x-amz-storage-class'] = opts[:redundancy] == :standard ? 'STANDARD' : 'REDUCED_REDUNDANCY'
67
+ headers['Content-Type'] = parse_content_type(opts[:content_type]) if opts[:content_type]
68
+
69
+ if opts.has_key?(:cache_control)
70
+ headers['Cache-Control'] = opts[:cache_control]
71
+ else
72
+ headers['Cache-Control'] = "max-age=#{opts[:ttl] || 5}"
73
+ end
74
+
75
+ @publish_queue.push({:key_name => file_name, :data => data, :headers => headers})
76
+ end
77
+
78
+ # Process queued uploads and push to S3
79
+ def run
80
+ threads = []
81
+ workers_to_use.times { threads << Thread.new { publish_from_queue } }
82
+ threads.each { |t| t.join }
83
+ true
84
+ end
85
+
86
+ private
87
+ def gzip data
88
+ gzipped_data = StringIO.open('', 'w+')
89
+
90
+ gzip_writer = Zlib::GzipWriter.new(gzipped_data)
91
+ gzip_writer.write(data)
92
+ gzip_writer.close
93
+
94
+ return gzipped_data.string
95
+ end
96
+
97
+ def parse_content_type content_type
98
+ case content_type
99
+ when :xml
100
+ 'application/xml'
101
+ when :text
102
+ 'text/plain'
103
+ when :html
104
+ 'text/html'
105
+ else
106
+ content_type
107
+ end
108
+ end
109
+
110
+ def publish_from_queue
111
+ loop do
112
+ item = @publish_queue.pop(true)
113
+
114
+ try_count = 0
115
+ begin
116
+ @s3.bucket(bucket_name).put(item[:key_name], item[:data], {}, 'public-read', item[:headers])
117
+ rescue Exception => e # backstop against transient S3 errors
118
+ raise e if try_count >= 1
119
+ try_count += 1
120
+ retry
121
+ end
122
+
123
+ logger << "Wrote http://#{bucket_name}.s3.amazonaws.com/#{item[:key_name]}"
124
+ end
125
+ rescue ThreadError # ThreadError hit when queue is empty. Simply jump out of loop and return to join().
126
+ end
127
+
128
+ end
@@ -0,0 +1,60 @@
1
+ # Generated by jeweler
2
+ # DO NOT EDIT THIS FILE DIRECTLY
3
+ # Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
4
+ # -*- encoding: utf-8 -*-
5
+
6
+ Gem::Specification.new do |s|
7
+ s.name = %q{s3-publisher}
8
+ s.version = "0.4.0"
9
+
10
+ s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
11
+ s.authors = ["Ben Koski"]
12
+ s.date = %q{2010-06-29}
13
+ s.description = %q{Publish data to S3 for the world to see}
14
+ s.email = %q{gems@benkoski.com}
15
+ s.extra_rdoc_files = [
16
+ "LICENSE",
17
+ "README.rdoc"
18
+ ]
19
+ s.files = [
20
+ ".document",
21
+ ".gitignore",
22
+ "LICENSE",
23
+ "README.rdoc",
24
+ "Rakefile",
25
+ "VERSION",
26
+ "lib/s3-publisher.rb",
27
+ "s3-publisher.gemspec",
28
+ "test/s3-publisher_test.rb",
29
+ "test/test_helper.rb"
30
+ ]
31
+ s.homepage = %q{http://github.com/bkoski/s3-publisher}
32
+ s.rdoc_options = ["--charset=UTF-8"]
33
+ s.require_paths = ["lib"]
34
+ s.rubygems_version = %q{1.3.5}
35
+ s.summary = %q{Publish data to S3 for the world to see}
36
+ s.test_files = [
37
+ "test/s3-publisher_test.rb",
38
+ "test/test_helper.rb"
39
+ ]
40
+
41
+ if s.respond_to? :specification_version then
42
+ current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
43
+ s.specification_version = 3
44
+
45
+ if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
46
+ s.add_development_dependency(%q<thoughtbot-shoulda>, [">= 0"])
47
+ s.add_runtime_dependency(%q<aws_credentials>, [">= 0"])
48
+ s.add_runtime_dependency(%q<right_aws>, ["= 2.0.0"])
49
+ else
50
+ s.add_dependency(%q<thoughtbot-shoulda>, [">= 0"])
51
+ s.add_dependency(%q<aws_credentials>, [">= 0"])
52
+ s.add_dependency(%q<right_aws>, ["= 2.0.0"])
53
+ end
54
+ else
55
+ s.add_dependency(%q<thoughtbot-shoulda>, [">= 0"])
56
+ s.add_dependency(%q<aws_credentials>, [">= 0"])
57
+ s.add_dependency(%q<right_aws>, ["= 2.0.0"])
58
+ end
59
+ end
60
+
@@ -0,0 +1,127 @@
1
+ require 'test_helper'
2
+
3
+ class S3PublisherTest < Test::Unit::TestCase
4
+
5
+ context "push" do
6
+
7
+ context "file_name" do
8
+ should "prepend base_path if provided on instantiate" do
9
+ set_put_expectation(:key_name => 'world_cup_2010/events.xml')
10
+ p = S3Publisher.new('test-bucket', :logger => Logger.new(nil), :base_path => 'world_cup_2010')
11
+ p.push('events.xml', '1234')
12
+ p.run
13
+ end
14
+
15
+ should "pass through unaltered if base_path not specified" do
16
+ set_put_expectation(:key_name => 'events.xml')
17
+ p = S3Publisher.new('test-bucket', :logger => Logger.new(nil))
18
+ p.push('events.xml', '1234')
19
+ p.run
20
+ end
21
+ end
22
+
23
+ context "gzip" do
24
+ should "gzip data if :gzip => true" do
25
+ set_put_expectation(:data => gzip('1234'))
26
+ push_test_data('myfile.txt', '1234', :gzip => true)
27
+ end
28
+
29
+ should "not gzip data if :gzip => false" do
30
+ set_put_expectation(:data => '1234')
31
+ push_test_data('myfile.txt', '1234', :gzip => false)
32
+ end
33
+
34
+ should "not gzip data if file ends in .jpg" do
35
+ set_put_expectation(:data => '1234')
36
+ push_test_data('myfile.jpg', '1234', {})
37
+ end
38
+
39
+ should "gzip data by default" do
40
+ set_put_expectation(:data => gzip('1234'))
41
+ push_test_data('myfile.txt', '1234', {})
42
+ end
43
+ end
44
+
45
+ context "redundancy" do
46
+ should "set REDUCED_REDUNDANCY by default" do
47
+ set_put_expectation(:headers => { 'x-amz-storage-class' => 'REDUCED_REDUNDANCY' })
48
+ push_test_data('myfile.txt', '1234', {})
49
+
50
+ end
51
+
52
+ should "set STANDARD if :redundancy => :standard is passed" do
53
+ set_put_expectation(:headers => { 'x-amz-storage-class' => 'STANDARD' })
54
+ push_test_data('myfile.txt', '1234', :redundancy => :standard)
55
+ end
56
+ end
57
+
58
+ context "content type" do
59
+ should "force Content-Type to user-supplied string if provided" do
60
+ set_put_expectation(:headers => { 'Content-Type' => 'audio/vorbis' })
61
+ push_test_data('myfile.txt', '1234', :content_type => 'audio/vorbis')
62
+ end
63
+
64
+ should "force Content-Type to application/xml if :xml provided" do
65
+ set_put_expectation(:headers => { 'Content-Type' => 'application/xml' })
66
+ push_test_data('myfile.txt', '1234', :content_type => :xml)
67
+ end
68
+
69
+ should "force Content-Type to text/plain if :text provided" do
70
+ set_put_expectation(:headers => { 'Content-Type' => 'text/plain' })
71
+ push_test_data('myfile.txt', '1234', :content_type => :text)
72
+ end
73
+
74
+ should "force Content-Type to text/html if :html provided" do
75
+ set_put_expectation(:headers => { 'Content-Type' => 'text/html' })
76
+ push_test_data('myfile.txt', '1234', :content_type => :html)
77
+ end
78
+ end
79
+
80
+ context "cache-control" do
81
+ should "set Cache-Control to user-supplied string if :cache_control provided" do
82
+ set_put_expectation(:headers => { 'Cache-Control' => 'private, max-age=0' })
83
+ push_test_data('myfile.txt', '1234', :cache_control => 'private, max-age=0')
84
+ end
85
+
86
+ should "set Cache-Control with :ttl provided" do
87
+ set_put_expectation(:headers => { 'Cache-Control' => 'max-age=55' })
88
+ push_test_data('myfile.txt', '1234', :ttl => 55)
89
+ end
90
+
91
+ should "set Cache-Control to a 5s ttl if no :ttl or :cache_control was provided" do
92
+ set_put_expectation(:headers => { 'Cache-Control' => 'max-age=5' })
93
+ push_test_data('myfile.txt', '1234', {})
94
+ end
95
+ end
96
+
97
+
98
+
99
+
100
+ end
101
+
102
+ def set_put_expectation opts
103
+ s3_stub = mock()
104
+ bucket_stub = mock()
105
+ bucket_stub.expects(:put).with(opts[:key_name] || anything, opts[:data] || anything, {}, 'public-read', opts[:headers] ? has_entries(opts[:headers]) : anything)
106
+
107
+ s3_stub.stubs(:bucket).returns(bucket_stub)
108
+ RightAws::S3.stubs(:new).returns(s3_stub)
109
+ end
110
+
111
+ def gzip data
112
+ gzipped_data = StringIO.open('', 'w+')
113
+
114
+ gzip_writer = Zlib::GzipWriter.new(gzipped_data)
115
+ gzip_writer.write(data)
116
+ gzip_writer.close
117
+
118
+ return gzipped_data.string
119
+ end
120
+
121
+ def push_test_data file_name, data, opts
122
+ p = S3Publisher.new('test-bucket', :logger => Logger.new(nil))
123
+ p.push(file_name, data, opts)
124
+ p.run
125
+ end
126
+
127
+ end
@@ -0,0 +1,11 @@
1
+ require 'rubygems'
2
+ require 'test/unit'
3
+ require 'shoulda'
4
+ require 'mocha'
5
+
6
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
7
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
8
+ require 's3-publisher'
9
+
10
+ class Test::Unit::TestCase
11
+ end
metadata ADDED
@@ -0,0 +1,95 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: s3-publisher
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.4.0
5
+ platform: ruby
6
+ authors:
7
+ - Ben Koski
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+
12
+ date: 2010-06-29 00:00:00 -04:00
13
+ default_executable:
14
+ dependencies:
15
+ - !ruby/object:Gem::Dependency
16
+ name: thoughtbot-shoulda
17
+ type: :development
18
+ version_requirement:
19
+ version_requirements: !ruby/object:Gem::Requirement
20
+ requirements:
21
+ - - ">="
22
+ - !ruby/object:Gem::Version
23
+ version: "0"
24
+ version:
25
+ - !ruby/object:Gem::Dependency
26
+ name: aws_credentials
27
+ type: :runtime
28
+ version_requirement:
29
+ version_requirements: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: "0"
34
+ version:
35
+ - !ruby/object:Gem::Dependency
36
+ name: right_aws
37
+ type: :runtime
38
+ version_requirement:
39
+ version_requirements: !ruby/object:Gem::Requirement
40
+ requirements:
41
+ - - "="
42
+ - !ruby/object:Gem::Version
43
+ version: 2.0.0
44
+ version:
45
+ description: Publish data to S3 for the world to see
46
+ email: gems@benkoski.com
47
+ executables: []
48
+
49
+ extensions: []
50
+
51
+ extra_rdoc_files:
52
+ - LICENSE
53
+ - README.rdoc
54
+ files:
55
+ - .document
56
+ - .gitignore
57
+ - LICENSE
58
+ - README.rdoc
59
+ - Rakefile
60
+ - VERSION
61
+ - lib/s3-publisher.rb
62
+ - s3-publisher.gemspec
63
+ - test/s3-publisher_test.rb
64
+ - test/test_helper.rb
65
+ has_rdoc: true
66
+ homepage: http://github.com/bkoski/s3-publisher
67
+ licenses: []
68
+
69
+ post_install_message:
70
+ rdoc_options:
71
+ - --charset=UTF-8
72
+ require_paths:
73
+ - lib
74
+ required_ruby_version: !ruby/object:Gem::Requirement
75
+ requirements:
76
+ - - ">="
77
+ - !ruby/object:Gem::Version
78
+ version: "0"
79
+ version:
80
+ required_rubygems_version: !ruby/object:Gem::Requirement
81
+ requirements:
82
+ - - ">="
83
+ - !ruby/object:Gem::Version
84
+ version: "0"
85
+ version:
86
+ requirements: []
87
+
88
+ rubyforge_project:
89
+ rubygems_version: 1.3.5
90
+ signing_key:
91
+ specification_version: 3
92
+ summary: Publish data to S3 for the world to see
93
+ test_files:
94
+ - test/s3-publisher_test.rb
95
+ - test/test_helper.rb