massive_sitemap-writer-s3 0.0.1.rc2 → 0.0.1.rc3

data/LICENCE

@@ -0,0 +1,26 @@
+Copyright (c) 2012, SoundCloud, Tobias Bielohlawek
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+- Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+- Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+- Neither the name of the SoundCloud nor the names of its contributors may be
+  used to endorse or promote products derived from this software without
+  specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -1,11 +1,43 @@
-# MassiveSitemap S3 Writer
-
-[![](http://travis-ci.org/rngtng/massive_sitemap-writer-s3.png)](http://travis-ci.org/rngtng/massive_sitemap-writer-s3)
+# MassiveSitemap S3 Writer [![](http://travis-ci.org/rngtng/massive_sitemap-writer-s3.png)](http://travis-ci.org/rngtng/massive_sitemap-writer-s3)
 
 S3 Writer extension for [MassiveSitemap](http://rngtng/massive_sitemap)
 
 ## Usage
 
+_S3 Writer_ extends `MassiveSitemap::Writer::GzipFile` for uploading the generated siemap file directly to
+S3. The state of the sitemap files is read from there too, which makes permanenty storage of the files localy obsolete.
+
+For initialization the S3 options are required as first parameter:
+
+```ruby
+require 'massive_sitemap'
+require 'massive_sitemap/writer/s3'
+
+s3_options = { :access_key_id => ..., :secret_access_key => ..., :bucket => ... }
+writer = MassiveSitemap::Writer::S3.new(s3_options)
+
+MassiveSitemap::generate(:url => ..., :writer => writer)
+
+```
+
 ## Dependencies
 
-* S3 gem
+Obviously depends on a S3 library which is the fabouly slick [S3 gem](https://github.com/qoobaa/s3).
+
+For further information check [Massive Sitemap gem](http://rngtng/massive_sitemap)
+
+
+## Contributing
+
+We'll check out your contribution if you:
+
+- Provide a comprehensive suite of tests for your fork.
+- Have a clear and documented rationale for your changes.
+- Package these up in a pull request.
+
+We'll do our best to help you out with any contribution issues you may have.
+
+
+## License
+
+The license is included as LICENSE in this directory.

data/VERSION

@@ -1 +1 @@
-0.0.1.rc2
+0.0.1.rc3

data/lib/massive_sitemap/writer/s3.rb

@@ -1,4 +1,7 @@
+# Copyright (c) 2012, SoundCloud Ltd., Tobias Bielohlawek
+
 require 's3'
+require 'retryable'
 require 'massive_sitemap/writer/gzip_file'
 
 module MassiveSitemap
@@ -16,10 +19,11 @@ module MassiveSitemap
         @filename = filename
         super
         # upload to amazon
-        # TODO what if fail??
-        @bucket.objects.build(::File.basename(@filename)).tap do |object|
-          object.content = ::File.open(@filename)
-          object.save
+        retryable( :tries => 3 ) do
+          @bucket.objects.build(::File.basename(@filename)).tap do |object|
+            object.content = ::File.open(@filename)
+            object.save
+          end
         end
       end
 

data/lib/massive_sitemap-writer-s3.rb

@@ -1 +1,3 @@
+# Copyright (c) 2012, SoundCloud Ltd., Tobias Bielohlawek
+
 require 'massive_sitemap/writer/s3'

data/massive_sitemap-writer-s3.gemspec

@@ -15,7 +15,11 @@ Gem::Specification.new do |s|
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
 
-  %w(s3 massive_sitemap).each do |gem|
+  s.extra_rdoc_files = [
+    "README.md"
+  ]
+
+  %w(s3 massive_sitemap retryable).each do |gem|
     s.add_runtime_dependency *gem.split(' ')
   end
 

data/spec/s3_spec.rb

@@ -1,24 +1,49 @@
 require "spec_helper"
+require "fileutils"
 
 describe MassiveSitemap::Writer::S3 do
-  let(:service) { }
-  let(:bucket) { "example" }
-  let(:writer) { MassiveSitemap::Writer::S3.new(service, bucket) }
-
-  it "works" do
-    # writer.close_stream(stream)
+  let(:object) { mock('object', :"content=" => [], :save => nil) }
+  let(:objects) { mock('objects', :find_all => [], :build => object) }
+  let(:bucket) { mock('bucket', :objects => objects) }
+  let(:service) { mock('service', :buckets => mock('buckets', :find =>  bucket)) }
+  let(:writer) { MassiveSitemap::Writer::S3.new(s3_options) }
+  let(:s3_options) { Hash.new(:access_key_id => "ak", :secret_access_key => "sa", :bucket => "bucket") }
 
+  before do
+    ::S3::Service.stub(:new).and_return(service)
   end
 
   describe "initialize" do
-    # @stream_ids.clear
-    it  "doesn't take streams ids from disk" do
+    it "creates service" do
+      ::S3::Service.should_receive(:new).with(s3_options).and_return(service)
+      writer
+    end
+
+    it "get S3 object service" do
+      bucket.should_receive(:objects).and_return(objects)
+      writer
     end
   end
 
   describe "close_stream" do
-     it "uploads to amazon" do
-     end
+    let(:filename) { "sitemap.xml.gz" }
+      before do
+        writer.init!
+      end
+
+      after do
+        FileUtils.rm(filename)
+      end
+
+      it "uploads to amazon" do
+        objects.should_receive(:build).with(filename).and_return(object)
+        writer.close!
+      end
+
+      it "retries on fails" do
+        object.should_receive(:"content=").once().and_raise
+        object.should_receive(:"content=").once().and_return(true)
+        writer.close!
+      end
   end
 end
-

data/spec/spec_helper.rb

@@ -1,3 +1,60 @@
 $:.unshift File.expand_path("../../lib", __FILE__)
 require 'bundler/setup'
 require 'massive_sitemap-writer-s3'
+
+
+  def setup
+    @buckets_list_body = <<-EOBuckets
+    <?xml version="1.0" encoding="UTF-8"?>\n<ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> <Owner> <ID>123u1odhkhfoadf</ID> <DisplayName>JohnDoe</DisplayName> </Owner> <Buckets> <Bucket> <Name>data.example.com</Name> <CreationDate>2009-07-02T11:56:58.000Z</CreationDate> </Bucket> <Bucket> <Name>images</Name> <CreationDate>2009-06-05T12:26:33.000Z</CreationDate> </Bucket> </Buckets> </ListAllMyBucketsResult>
+    EOBuckets
+
+    @bucket_not_exists = <<-EOBucketNoexists
+    <?xml version="1.0" encoding="UTF-8"?>\n<Error> <Code>NoSuchBucket</Code> <Message>The specified bucket does not exists</Message> <BucketName>data2.example.com</BucketName> <RequestId>8D7519AAE74E9E99</RequestId> <HostId>DvKnnNSMnPHd1oXukyRaFNv8Lg/bpwhuUtY8Kj7eDLbaIrIT8JebSnHwi89AK1P+</HostId> </Error>
+    EOBucketNoexists
+
+    @bucket_exists = <<-EOBucketexists
+    <?xml version="1.0" encoding="UTF-8"?>\n<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> <Name>data.synergypeople.net</Name> <Prefix></Prefix> <Marker></Marker> <MaxKeys>1000</MaxKeys> <IsTruncated>false</IsTruncated> </ListBucketResult>
+    EOBucketexists
+
+    @service_empty_buckets_list = S3::Service.new(
+      :access_key_id =>  "12345678901234567890",
+      :secret_access_key =>  "qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDF"
+    )
+    @response_empty_buckets_list = Net::HTTPOK.new("1.1", "200", "OK")
+    @service_empty_buckets_list.stubs(:service_request).returns(@response_empty_buckets_list)
+    @response_empty_buckets_list.stubs(:body).returns(@buckets_empty_list_body)
+
+    @service_buckets_list = S3::Service.new(
+      :access_key_id =>  "12345678901234567890",
+      :secret_access_key =>  "qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDF"
+    )
+    @response_buckets_list = Net::HTTPOK.new("1.1", "200", "OK")
+    @service_buckets_list.stubs(:service_request).returns(@response_buckets_list)
+    @response_buckets_list.stubs(:body).returns(@buckets_list_body)
+
+    @service_bucket_exists = S3::Service.new(
+      :access_key_id =>  "12345678901234567890",
+      :secret_access_key =>  "qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDF"
+    )
+    @response_bucket_exists = Net::HTTPNotFound.new("1.1", "200", "OK")
+    @service_bucket_exists.stubs(:service_request).returns(@response_bucket_exists)
+    @response_bucket_exists.stubs(:body).returns(@bucket_exists)
+
+    @service_bucket_not_exists = S3::Service.new(
+      :access_key_id =>  "12345678901234567890",
+      :secret_access_key =>  "qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDF"
+    )
+    @response_bucket_not_exists = Net::HTTPNotFound.new("1.1", "404", "Not Found")
+    @service_bucket_not_exists.stubs(:service_request).raises(S3::Error::NoSuchBucket.new(404, @response_bucket_not_exists))
+    @response_bucket_not_exists.stubs(:body).returns(@bucket_not_exists)
+
+    @buckets_empty_list = []
+    @buckets_empty_list_body = <<-EOEmptyBuckets
+    <?xml version="1.0" encoding="UTF-8"?>\n<ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> <Owner> <ID>123u1odhkhfoadf</ID> <DisplayName>JohnDoe</DisplayName> </Owner> <Buckets> </Buckets> </ListAllMyBucketsResult>
+    EOEmptyBuckets
+
+    @buckets_list = [
+      S3::Bucket.send(:new, @service_buckets_list, "data.example.com"),
+      S3::Bucket.send(:new, @service_buckets_list, "images")
+    ]
+  end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: massive_sitemap-writer-s3
 version: !ruby/object:Gem::Version
-  version: 0.0.1.rc2
+  version: 0.0.1.rc3
   prerelease: 6
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-15 00:00:00.000000000Z
+date: 2012-02-16 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: s3
-  requirement: &70292589728700 !ruby/object:Gem::Requirement
+  requirement: &70254065150480 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70292589728700
+  version_requirements: *70254065150480
 - !ruby/object:Gem::Dependency
   name: massive_sitemap
-  requirement: &70292589728240 !ruby/object:Gem::Requirement
+  requirement: &70254065150020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,21 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70292589728240
+  version_requirements: *70254065150020
+- !ruby/object:Gem::Dependency
+  name: retryable
+  requirement: &70254065149560 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70254065149560
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70292589727720 !ruby/object:Gem::Requirement
+  requirement: &70254068290660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +54,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70292589727720
+  version_requirements: *70254068290660
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70292589727260 !ruby/object:Gem::Requirement
+  requirement: &70254068290200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,18 +65,20 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70292589727260
+  version_requirements: *70254068290200
 description: Amazon S3 Writer extension for MassiveSitemap
 email:
 - tobi@soundcloud.com
 executables: []
 extensions: []
-extra_rdoc_files: []
+extra_rdoc_files:
+- README.md
 files:
 - .gitignore
 - .travis.yml
 - CHANGELOG.md
 - Gemfile
+- LICENCE
 - README.md
 - Rakefile
 - VERSION