s3-publisher 0.4.0

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,5 @@
+*.sw?
+.DS_Store
+coverage
+rdoc
+pkg

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Ben Koski
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,38 @@
+= s3-publisher
+
+S3Publisher is meant as a clean, simple, sensible-defaults way to publish
+files to Amazon S3 for the world to see.
+
+Basic usage:
+
+    require 's3-publisher'
+    S3Publisher.publish('my-bucket') do |p|
+        p.push('test.txt', 'abc1234')
+    end
+
+This will:
+* push test.txt to my-bucket.s3.amazonaws.com
+* use reduced redundancy storage
+* set security to public-read
+* gzip contents ('abc1234') and set a Content-Encoding: gzip header so clients know to decompress
+* set a Cache-Control: max-age=5 header
+
+Slightly more advanced example:
+
+    S3Publisher.publish('my-bucket', :base_path => 'world_cup') do |p|
+        p.push('events.xml', '<xml>...', :ttl => 15)
+    end
+
+In this example:
+* file will be written to my-bucket.s3.amazonaws.com/world_cup/events.xml
+* Cache-Control: max-age=15 will be set
+  
+A few miscellaneous notes:
+* gzip compress is skipped on .jpg/gif/png/tif files
+* uploads are multi-threaded.  You can control worker thread count on instantiation.
+
+See class docs for further options.
+
+== Copyright
+
+Copyright (c) 2010 Ben Koski. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,58 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "s3-publisher"
+    gem.summary = %Q{Publish data to S3 for the world to see}
+    gem.description = %Q{Publish data to S3 for the world to see}
+    gem.email = "gems@benkoski.com"
+    gem.homepage = "http://github.com/bkoski/s3-publisher"
+    gem.authors = ["Ben Koski"]
+    gem.add_development_dependency "thoughtbot-shoulda"
+    gem.add_dependency 'aws_credentials'
+    gem.add_dependency 'right_aws', '=2.0.0'
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*_test.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/*_test.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+task :test => :check_dependencies
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  if File.exist?('VERSION')
+    version = File.read('VERSION')
+  else
+    version = ""
+  end
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "s3-publisher #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.4.0

data/lib/s3-publisher.rb

@@ -0,0 +1,128 @@
+require 'rubygems'
+
+gem 'right_aws', '=2.0.0'
+require 'right_aws'
+
+require 'aws_credentials'
+require 'zlib'
+require 'thread'
+
+Thread.abort_on_exception = true
+
+# You can either use the block syntax, or:
+# * instantiate a class
+# * queue data to be published with push
+# * call run to actually upload the data to S3
+class S3Publisher
+  
+  attr_reader :bucket_name, :base_path, :logger, :workers_to_use
+
+  # Block style.  run is called for you on block close.
+  #  S3Publisher.publish('my-bucket') do |p|
+  #    p.push('test.txt', '123abc')
+  #  end
+  def self.publish bucket_name, opts={}, &block
+    p = self.new(bucket_name, opts)
+    yield(p)
+    p.run
+  end
+
+  # Pass the publisher a bucket_name along with any of the following options:
+  # * <tt>base_path</tt> - path prepended to supplied file_name on upload
+  # * <tt>logger</tt> - a logger object to recieve 'uploaded' messages.  Defaults to STDOUT.
+  # * <tt>workers</tt> - number of threads to use when pushing to S3. Defaults to 3.
+  def initialize bucket_name, opts={}
+    @s3 =  RightAws::S3.new(AWSCredentials.access_key, AWSCredentials.secret_access_key, :multi_thread => true,
+                                                                                         :protocol => 'http',
+                                                                                         :port => 80, 
+                                                                                         :logger => Logger.new(nil))
+    @bucket_name, @base_path = bucket_name, opts[:base_path]
+    @logger = opts[:logger] || STDOUT
+    @workers_to_use = opts[:workers] || 3
+    @publish_queue = Queue.new
+  end
+  
+  # Pass:
+  # * <tt>file_name</tt> - name of file on S3. base_path will be prepended if supplied on instantiate.
+  # * <tt>data</tt> - data to be uploaded as a string
+  #
+  # And one or many options:
+  # * <tt>:gzip (true|false)</tt> - gzip file contents?  defaults to true.
+  # * <tt>:ttl</tt> - TTL in seconds for cache-control header. defaults to 5.
+  # * <tt>:cache_control</tt> - specify Cache-Control header directly if you don't like the default
+  # * <tt>:content_type</tt> - no need to specify if default based on extension is okay.  But if you need to force,
+  #   you can provide :xml, :html, :text, or your own custom string.
+  # * <tt>:redundancy</tt> - by default objects are stored at reduced redundancy, pass :standard to store at full
+  def push file_name, data, opts={}
+    headers = {}
+    
+    file_name = "#{base_path}/#{file_name}" unless base_path.nil?
+    
+    unless opts[:gzip] == false || file_name.match(/\.(jpg|gif|png|tif)$/)
+      data = gzip(data)
+      headers['Content-Encoding'] = 'gzip'
+    end
+    
+    headers['x-amz-storage-class'] = opts[:redundancy] == :standard ? 'STANDARD' : 'REDUCED_REDUNDANCY' 
+    headers['Content-Type']        = parse_content_type(opts[:content_type])  if opts[:content_type]
+
+    if opts.has_key?(:cache_control)
+      headers['Cache-Control'] = opts[:cache_control]
+    else
+      headers['Cache-Control'] = "max-age=#{opts[:ttl] || 5}"
+    end
+
+    @publish_queue.push({:key_name => file_name, :data => data, :headers => headers})
+  end  
+    
+  # Process queued uploads and push to S3
+  def run
+    threads = []
+    workers_to_use.times { threads << Thread.new { publish_from_queue } }
+    threads.each { |t| t.join }
+    true
+  end
+  
+  private
+  def gzip data
+    gzipped_data = StringIO.open('', 'w+')
+    
+    gzip_writer = Zlib::GzipWriter.new(gzipped_data)
+    gzip_writer.write(data)
+    gzip_writer.close
+    
+    return gzipped_data.string
+  end
+    
+  def parse_content_type content_type
+    case content_type
+    when :xml
+      'application/xml'
+    when :text
+      'text/plain'
+    when :html
+      'text/html'
+    else
+      content_type
+    end
+  end
+  
+  def publish_from_queue
+    loop do
+      item = @publish_queue.pop(true)
+    
+      try_count = 0
+      begin
+        @s3.bucket(bucket_name).put(item[:key_name], item[:data], {}, 'public-read', item[:headers])
+      rescue Exception => e # backstop against transient S3 errors
+        raise e if try_count >= 1
+        try_count += 1
+        retry
+      end
+    
+      logger << "Wrote http://#{bucket_name}.s3.amazonaws.com/#{item[:key_name]}"
+    end
+  rescue ThreadError  # ThreadError hit when queue is empty.  Simply jump out of loop and return to join().
+  end
+  
+end

data/s3-publisher.gemspec

@@ -0,0 +1,60 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{s3-publisher}
+  s.version = "0.4.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Ben Koski"]
+  s.date = %q{2010-06-29}
+  s.description = %q{Publish data to S3 for the world to see}
+  s.email = %q{gems@benkoski.com}
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+     ".gitignore",
+     "LICENSE",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION",
+     "lib/s3-publisher.rb",
+     "s3-publisher.gemspec",
+     "test/s3-publisher_test.rb",
+     "test/test_helper.rb"
+  ]
+  s.homepage = %q{http://github.com/bkoski/s3-publisher}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.5}
+  s.summary = %q{Publish data to S3 for the world to see}
+  s.test_files = [
+    "test/s3-publisher_test.rb",
+     "test/test_helper.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<thoughtbot-shoulda>, [">= 0"])
+      s.add_runtime_dependency(%q<aws_credentials>, [">= 0"])
+      s.add_runtime_dependency(%q<right_aws>, ["= 2.0.0"])
+    else
+      s.add_dependency(%q<thoughtbot-shoulda>, [">= 0"])
+      s.add_dependency(%q<aws_credentials>, [">= 0"])
+      s.add_dependency(%q<right_aws>, ["= 2.0.0"])
+    end
+  else
+    s.add_dependency(%q<thoughtbot-shoulda>, [">= 0"])
+    s.add_dependency(%q<aws_credentials>, [">= 0"])
+    s.add_dependency(%q<right_aws>, ["= 2.0.0"])
+  end
+end
+

data/test/s3-publisher_test.rb

@@ -0,0 +1,127 @@
+require 'test_helper'
+
+class S3PublisherTest < Test::Unit::TestCase
+  
+  context "push" do
+           
+    context "file_name" do
+      should "prepend base_path if provided on instantiate" do
+        set_put_expectation(:key_name => 'world_cup_2010/events.xml')
+        p = S3Publisher.new('test-bucket', :logger => Logger.new(nil), :base_path => 'world_cup_2010')
+        p.push('events.xml', '1234')
+        p.run
+      end
+      
+      should "pass through unaltered if base_path not specified" do
+        set_put_expectation(:key_name => 'events.xml')
+        p = S3Publisher.new('test-bucket', :logger => Logger.new(nil))
+        p.push('events.xml', '1234')
+        p.run
+      end      
+    end
+    
+    context "gzip" do
+      should "gzip data if :gzip => true" do
+        set_put_expectation(:data => gzip('1234'))
+        push_test_data('myfile.txt', '1234', :gzip => true)
+      end
+      
+      should "not gzip data if :gzip => false" do
+        set_put_expectation(:data => '1234')
+        push_test_data('myfile.txt', '1234', :gzip => false)
+      end
+
+      should "not gzip data if file ends in .jpg" do
+        set_put_expectation(:data => '1234')
+        push_test_data('myfile.jpg', '1234', {})
+      end
+      
+      should "gzip data by default" do
+        set_put_expectation(:data => gzip('1234'))
+        push_test_data('myfile.txt', '1234', {})
+      end
+    end
+    
+    context "redundancy" do
+      should "set REDUCED_REDUNDANCY by default" do
+        set_put_expectation(:headers => { 'x-amz-storage-class' => 'REDUCED_REDUNDANCY' })
+        push_test_data('myfile.txt', '1234', {})
+
+      end
+      
+      should "set STANDARD if :redundancy => :standard is passed" do
+        set_put_expectation(:headers => { 'x-amz-storage-class' => 'STANDARD' })
+        push_test_data('myfile.txt', '1234', :redundancy => :standard)
+      end
+    end
+
+    context "content type" do
+      should "force Content-Type to user-supplied string if provided" do
+        set_put_expectation(:headers => { 'Content-Type' => 'audio/vorbis' })
+        push_test_data('myfile.txt', '1234', :content_type => 'audio/vorbis')
+      end
+      
+      should "force Content-Type to application/xml if :xml provided" do
+        set_put_expectation(:headers => { 'Content-Type' => 'application/xml' })
+        push_test_data('myfile.txt', '1234', :content_type => :xml)
+      end
+      
+      should "force Content-Type to text/plain if :text provided" do
+        set_put_expectation(:headers => { 'Content-Type' => 'text/plain' })
+        push_test_data('myfile.txt', '1234', :content_type => :text)
+      end
+      
+      should "force Content-Type to text/html if :html provided" do
+        set_put_expectation(:headers => { 'Content-Type' => 'text/html' })
+        push_test_data('myfile.txt', '1234', :content_type => :html)
+      end
+    end
+
+    context "cache-control" do
+      should "set Cache-Control to user-supplied string if :cache_control provided" do
+        set_put_expectation(:headers => { 'Cache-Control' => 'private, max-age=0' })
+        push_test_data('myfile.txt', '1234', :cache_control => 'private, max-age=0')
+      end
+      
+      should "set Cache-Control with :ttl provided" do
+        set_put_expectation(:headers => { 'Cache-Control' => 'max-age=55' })
+        push_test_data('myfile.txt', '1234', :ttl => 55)
+      end
+      
+      should "set Cache-Control to a 5s ttl if no :ttl or :cache_control was provided" do
+        set_put_expectation(:headers => { 'Cache-Control' => 'max-age=5' })
+        push_test_data('myfile.txt', '1234', {})
+      end
+    end
+    
+    
+    
+    
+  end
+  
+  def set_put_expectation opts
+    s3_stub = mock()
+    bucket_stub = mock()
+    bucket_stub.expects(:put).with(opts[:key_name] || anything, opts[:data] || anything, {}, 'public-read', opts[:headers] ? has_entries(opts[:headers]) : anything)
+    
+    s3_stub.stubs(:bucket).returns(bucket_stub)
+    RightAws::S3.stubs(:new).returns(s3_stub)
+  end
+  
+  def gzip data
+    gzipped_data = StringIO.open('', 'w+')
+    
+    gzip_writer = Zlib::GzipWriter.new(gzipped_data)
+    gzip_writer.write(data)
+    gzip_writer.close
+    
+    return gzipped_data.string
+  end
+  
+  def push_test_data file_name, data, opts
+    p = S3Publisher.new('test-bucket', :logger => Logger.new(nil))
+    p.push(file_name, data, opts)
+    p.run
+  end
+  
+end

data/test/test_helper.rb

@@ -0,0 +1,11 @@
+require 'rubygems'
+require 'test/unit'
+require 'shoulda'
+require 'mocha'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 's3-publisher'
+
+class Test::Unit::TestCase
+end

metadata

@@ -0,0 +1,95 @@
+--- !ruby/object:Gem::Specification 
+name: s3-publisher
+version: !ruby/object:Gem::Version 
+  version: 0.4.0
+platform: ruby
+authors: 
+- Ben Koski
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-06-29 00:00:00 -04:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: thoughtbot-shoulda
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: aws_credentials
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: right_aws
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        version: 2.0.0
+    version: 
+description: Publish data to S3 for the world to see
+email: gems@benkoski.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- .document
+- .gitignore
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/s3-publisher.rb
+- s3-publisher.gemspec
+- test/s3-publisher_test.rb
+- test/test_helper.rb
+has_rdoc: true
+homepage: http://github.com/bkoski/s3-publisher
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Publish data to S3 for the world to see
+test_files: 
+- test/s3-publisher_test.rb
+- test/test_helper.rb