massive_sitemap-writer-s3 0.0.1.rc2 → 0.0.1.rc3

Sign up to get free protection for your applications and to get access to all the features.
data/LICENCE ADDED
@@ -0,0 +1,26 @@
1
+ Copyright (c) 2012, SoundCloud, Tobias Bielohlawek
2
+
3
+ All rights reserved.
4
+
5
+ Redistribution and use in source and binary forms, with or without
6
+ modification, are permitted provided that the following conditions are met:
7
+
8
+ - Redistributions of source code must retain the above copyright notice, this
9
+ list of conditions and the following disclaimer.
10
+ - Redistributions in binary form must reproduce the above copyright notice,
11
+ this list of conditions and the following disclaimer in the documentation
12
+ and/or other materials provided with the distribution.
13
+ - Neither the name of the SoundCloud nor the names of its contributors may be
14
+ used to endorse or promote products derived from this software without
15
+ specific prior written permission.
16
+
17
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
20
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
21
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
23
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
24
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
25
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
data/README.md CHANGED
@@ -1,11 +1,43 @@
1
- # MassiveSitemap S3 Writer
2
-
3
- [![](http://travis-ci.org/rngtng/massive_sitemap-writer-s3.png)](http://travis-ci.org/rngtng/massive_sitemap-writer-s3)
1
+ # MassiveSitemap S3 Writer [![](http://travis-ci.org/rngtng/massive_sitemap-writer-s3.png)](http://travis-ci.org/rngtng/massive_sitemap-writer-s3)
4
2
 
5
3
  S3 Writer extension for [MassiveSitemap](http://rngtng/massive_sitemap)
6
4
 
7
5
  ## Usage
8
6
 
7
+ _S3 Writer_ extends `MassiveSitemap::Writer::GzipFile` for uploading the generated siemap file directly to
8
+ S3. The state of the sitemap files is read from there too, which makes permanenty storage of the files localy obsolete.
9
+
10
+ For initialization the S3 options are required as first parameter:
11
+
12
+ ```ruby
13
+ require 'massive_sitemap'
14
+ require 'massive_sitemap/writer/s3'
15
+
16
+ s3_options = { :access_key_id => ..., :secret_access_key => ..., :bucket => ... }
17
+ writer = MassiveSitemap::Writer::S3.new(s3_options)
18
+
19
+ MassiveSitemap::generate(:url => ..., :writer => writer)
20
+
21
+ ```
22
+
9
23
  ## Dependencies
10
24
 
11
- * S3 gem
25
+ Obviously depends on a S3 library which is the fabouly slick [S3 gem](https://github.com/qoobaa/s3).
26
+
27
+ For further information check [Massive Sitemap gem](http://rngtng/massive_sitemap)
28
+
29
+
30
+ ## Contributing
31
+
32
+ We'll check out your contribution if you:
33
+
34
+ - Provide a comprehensive suite of tests for your fork.
35
+ - Have a clear and documented rationale for your changes.
36
+ - Package these up in a pull request.
37
+
38
+ We'll do our best to help you out with any contribution issues you may have.
39
+
40
+
41
+ ## License
42
+
43
+ The license is included as LICENSE in this directory.
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.0.1.rc2
1
+ 0.0.1.rc3
@@ -1,4 +1,7 @@
1
+ # Copyright (c) 2012, SoundCloud Ltd., Tobias Bielohlawek
2
+
1
3
  require 's3'
4
+ require 'retryable'
2
5
  require 'massive_sitemap/writer/gzip_file'
3
6
 
4
7
  module MassiveSitemap
@@ -16,10 +19,11 @@ module MassiveSitemap
16
19
  @filename = filename
17
20
  super
18
21
  # upload to amazon
19
- # TODO what if fail??
20
- @bucket.objects.build(::File.basename(@filename)).tap do |object|
21
- object.content = ::File.open(@filename)
22
- object.save
22
+ retryable( :tries => 3 ) do
23
+ @bucket.objects.build(::File.basename(@filename)).tap do |object|
24
+ object.content = ::File.open(@filename)
25
+ object.save
26
+ end
23
27
  end
24
28
  end
25
29
 
@@ -1 +1,3 @@
1
+ # Copyright (c) 2012, SoundCloud Ltd., Tobias Bielohlawek
2
+
1
3
  require 'massive_sitemap/writer/s3'
@@ -15,7 +15,11 @@ Gem::Specification.new do |s|
15
15
  s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
16
16
  s.require_paths = ["lib"]
17
17
 
18
- %w(s3 massive_sitemap).each do |gem|
18
+ s.extra_rdoc_files = [
19
+ "README.md"
20
+ ]
21
+
22
+ %w(s3 massive_sitemap retryable).each do |gem|
19
23
  s.add_runtime_dependency *gem.split(' ')
20
24
  end
21
25
 
data/spec/s3_spec.rb CHANGED
@@ -1,24 +1,49 @@
1
1
  require "spec_helper"
2
+ require "fileutils"
2
3
 
3
4
  describe MassiveSitemap::Writer::S3 do
4
- let(:service) { }
5
- let(:bucket) { "example" }
6
- let(:writer) { MassiveSitemap::Writer::S3.new(service, bucket) }
7
-
8
- it "works" do
9
- # writer.close_stream(stream)
5
+ let(:object) { mock('object', :"content=" => [], :save => nil) }
6
+ let(:objects) { mock('objects', :find_all => [], :build => object) }
7
+ let(:bucket) { mock('bucket', :objects => objects) }
8
+ let(:service) { mock('service', :buckets => mock('buckets', :find => bucket)) }
9
+ let(:writer) { MassiveSitemap::Writer::S3.new(s3_options) }
10
+ let(:s3_options) { Hash.new(:access_key_id => "ak", :secret_access_key => "sa", :bucket => "bucket") }
10
11
 
12
+ before do
13
+ ::S3::Service.stub(:new).and_return(service)
11
14
  end
12
15
 
13
16
  describe "initialize" do
14
- # @stream_ids.clear
15
- it "doesn't take streams ids from disk" do
17
+ it "creates service" do
18
+ ::S3::Service.should_receive(:new).with(s3_options).and_return(service)
19
+ writer
20
+ end
21
+
22
+ it "get S3 object service" do
23
+ bucket.should_receive(:objects).and_return(objects)
24
+ writer
16
25
  end
17
26
  end
18
27
 
19
28
  describe "close_stream" do
20
- it "uploads to amazon" do
21
- end
29
+ let(:filename) { "sitemap.xml.gz" }
30
+ before do
31
+ writer.init!
32
+ end
33
+
34
+ after do
35
+ FileUtils.rm(filename)
36
+ end
37
+
38
+ it "uploads to amazon" do
39
+ objects.should_receive(:build).with(filename).and_return(object)
40
+ writer.close!
41
+ end
42
+
43
+ it "retries on fails" do
44
+ object.should_receive(:"content=").once().and_raise
45
+ object.should_receive(:"content=").once().and_return(true)
46
+ writer.close!
47
+ end
22
48
  end
23
49
  end
24
-
data/spec/spec_helper.rb CHANGED
@@ -1,3 +1,60 @@
1
1
  $:.unshift File.expand_path("../../lib", __FILE__)
2
2
  require 'bundler/setup'
3
3
  require 'massive_sitemap-writer-s3'
4
+
5
+
6
+ def setup
7
+ @buckets_list_body = <<-EOBuckets
8
+ <?xml version="1.0" encoding="UTF-8"?>\n<ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> <Owner> <ID>123u1odhkhfoadf</ID> <DisplayName>JohnDoe</DisplayName> </Owner> <Buckets> <Bucket> <Name>data.example.com</Name> <CreationDate>2009-07-02T11:56:58.000Z</CreationDate> </Bucket> <Bucket> <Name>images</Name> <CreationDate>2009-06-05T12:26:33.000Z</CreationDate> </Bucket> </Buckets> </ListAllMyBucketsResult>
9
+ EOBuckets
10
+
11
+ @bucket_not_exists = <<-EOBucketNoexists
12
+ <?xml version="1.0" encoding="UTF-8"?>\n<Error> <Code>NoSuchBucket</Code> <Message>The specified bucket does not exists</Message> <BucketName>data2.example.com</BucketName> <RequestId>8D7519AAE74E9E99</RequestId> <HostId>DvKnnNSMnPHd1oXukyRaFNv8Lg/bpwhuUtY8Kj7eDLbaIrIT8JebSnHwi89AK1P+</HostId> </Error>
13
+ EOBucketNoexists
14
+
15
+ @bucket_exists = <<-EOBucketexists
16
+ <?xml version="1.0" encoding="UTF-8"?>\n<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> <Name>data.synergypeople.net</Name> <Prefix></Prefix> <Marker></Marker> <MaxKeys>1000</MaxKeys> <IsTruncated>false</IsTruncated> </ListBucketResult>
17
+ EOBucketexists
18
+
19
+ @service_empty_buckets_list = S3::Service.new(
20
+ :access_key_id => "12345678901234567890",
21
+ :secret_access_key => "qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDF"
22
+ )
23
+ @response_empty_buckets_list = Net::HTTPOK.new("1.1", "200", "OK")
24
+ @service_empty_buckets_list.stubs(:service_request).returns(@response_empty_buckets_list)
25
+ @response_empty_buckets_list.stubs(:body).returns(@buckets_empty_list_body)
26
+
27
+ @service_buckets_list = S3::Service.new(
28
+ :access_key_id => "12345678901234567890",
29
+ :secret_access_key => "qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDF"
30
+ )
31
+ @response_buckets_list = Net::HTTPOK.new("1.1", "200", "OK")
32
+ @service_buckets_list.stubs(:service_request).returns(@response_buckets_list)
33
+ @response_buckets_list.stubs(:body).returns(@buckets_list_body)
34
+
35
+ @service_bucket_exists = S3::Service.new(
36
+ :access_key_id => "12345678901234567890",
37
+ :secret_access_key => "qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDF"
38
+ )
39
+ @response_bucket_exists = Net::HTTPNotFound.new("1.1", "200", "OK")
40
+ @service_bucket_exists.stubs(:service_request).returns(@response_bucket_exists)
41
+ @response_bucket_exists.stubs(:body).returns(@bucket_exists)
42
+
43
+ @service_bucket_not_exists = S3::Service.new(
44
+ :access_key_id => "12345678901234567890",
45
+ :secret_access_key => "qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDF"
46
+ )
47
+ @response_bucket_not_exists = Net::HTTPNotFound.new("1.1", "404", "Not Found")
48
+ @service_bucket_not_exists.stubs(:service_request).raises(S3::Error::NoSuchBucket.new(404, @response_bucket_not_exists))
49
+ @response_bucket_not_exists.stubs(:body).returns(@bucket_not_exists)
50
+
51
+ @buckets_empty_list = []
52
+ @buckets_empty_list_body = <<-EOEmptyBuckets
53
+ <?xml version="1.0" encoding="UTF-8"?>\n<ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> <Owner> <ID>123u1odhkhfoadf</ID> <DisplayName>JohnDoe</DisplayName> </Owner> <Buckets> </Buckets> </ListAllMyBucketsResult>
54
+ EOEmptyBuckets
55
+
56
+ @buckets_list = [
57
+ S3::Bucket.send(:new, @service_buckets_list, "data.example.com"),
58
+ S3::Bucket.send(:new, @service_buckets_list, "images")
59
+ ]
60
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: massive_sitemap-writer-s3
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.1.rc2
4
+ version: 0.0.1.rc3
5
5
  prerelease: 6
6
6
  platform: ruby
7
7
  authors:
@@ -9,11 +9,11 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-02-15 00:00:00.000000000Z
12
+ date: 2012-02-16 00:00:00.000000000Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: s3
16
- requirement: &70292589728700 !ruby/object:Gem::Requirement
16
+ requirement: &70254065150480 !ruby/object:Gem::Requirement
17
17
  none: false
18
18
  requirements:
19
19
  - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
21
21
  version: '0'
22
22
  type: :runtime
23
23
  prerelease: false
24
- version_requirements: *70292589728700
24
+ version_requirements: *70254065150480
25
25
  - !ruby/object:Gem::Dependency
26
26
  name: massive_sitemap
27
- requirement: &70292589728240 !ruby/object:Gem::Requirement
27
+ requirement: &70254065150020 !ruby/object:Gem::Requirement
28
28
  none: false
29
29
  requirements:
30
30
  - - ! '>='
@@ -32,10 +32,21 @@ dependencies:
32
32
  version: '0'
33
33
  type: :runtime
34
34
  prerelease: false
35
- version_requirements: *70292589728240
35
+ version_requirements: *70254065150020
36
+ - !ruby/object:Gem::Dependency
37
+ name: retryable
38
+ requirement: &70254065149560 !ruby/object:Gem::Requirement
39
+ none: false
40
+ requirements:
41
+ - - ! '>='
42
+ - !ruby/object:Gem::Version
43
+ version: '0'
44
+ type: :runtime
45
+ prerelease: false
46
+ version_requirements: *70254065149560
36
47
  - !ruby/object:Gem::Dependency
37
48
  name: rake
38
- requirement: &70292589727720 !ruby/object:Gem::Requirement
49
+ requirement: &70254068290660 !ruby/object:Gem::Requirement
39
50
  none: false
40
51
  requirements:
41
52
  - - ! '>='
@@ -43,10 +54,10 @@ dependencies:
43
54
  version: '0'
44
55
  type: :development
45
56
  prerelease: false
46
- version_requirements: *70292589727720
57
+ version_requirements: *70254068290660
47
58
  - !ruby/object:Gem::Dependency
48
59
  name: rspec
49
- requirement: &70292589727260 !ruby/object:Gem::Requirement
60
+ requirement: &70254068290200 !ruby/object:Gem::Requirement
50
61
  none: false
51
62
  requirements:
52
63
  - - ! '>='
@@ -54,18 +65,20 @@ dependencies:
54
65
  version: '0'
55
66
  type: :development
56
67
  prerelease: false
57
- version_requirements: *70292589727260
68
+ version_requirements: *70254068290200
58
69
  description: Amazon S3 Writer extension for MassiveSitemap
59
70
  email:
60
71
  - tobi@soundcloud.com
61
72
  executables: []
62
73
  extensions: []
63
- extra_rdoc_files: []
74
+ extra_rdoc_files:
75
+ - README.md
64
76
  files:
65
77
  - .gitignore
66
78
  - .travis.yml
67
79
  - CHANGELOG.md
68
80
  - Gemfile
81
+ - LICENCE
69
82
  - README.md
70
83
  - Rakefile
71
84
  - VERSION