massive_sitemap 2.0.0.rc2 → 2.0.0.rc3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG.md +1 -1
- data/README.md +7 -0
- data/VERSION +1 -1
- data/lib/massive_sitemap/builder/base.rb +11 -12
- data/lib/massive_sitemap/builder/index.rb +9 -0
- data/lib/massive_sitemap/builder/rotating.rb +8 -5
- data/lib/massive_sitemap/builder.rb +1 -1
- data/lib/massive_sitemap/ping.rb +22 -14
- data/lib/massive_sitemap/writer/file.rb +7 -6
- data/lib/massive_sitemap/writer/gzip_file.rb +1 -1
- data/lib/massive_sitemap.rb +14 -14
- data/spec/builder/base_spec.rb +3 -3
- data/spec/builder/index_spec.rb +6 -2
- data/spec/massive_sitemap_spec.rb +117 -95
- data/spec/ping_spec.rb +32 -0
- data/spec/writer/file_spec.rb +17 -12
- data/spec/writer/gzip_file_spec.rb +4 -4
- metadata +7 -5
data/CHANGELOG.md
CHANGED
|
@@ -4,7 +4,6 @@
|
|
|
4
4
|
|
|
5
5
|
## v2.0.x - ???
|
|
6
6
|
|
|
7
|
-
* updated/fixed Ping
|
|
8
7
|
* updated Docu
|
|
9
8
|
* switch to writer chain
|
|
10
9
|
* add BigSitemap API
|
|
@@ -26,3 +25,4 @@
|
|
|
26
25
|
* move index build into indexer and resource handling/selection into writer
|
|
27
26
|
* manifest handling:
|
|
28
27
|
* moved Amazon S3 integration to [massive_sitemap-writer-s3](https://github.com/rngtng/massive_sitemap-writer-s3)
|
|
28
|
+
* updated/fixed Ping
|
data/README.md
CHANGED
|
@@ -9,6 +9,13 @@ It implements various generation stategies, e.g. to split large Sitemaps into mu
|
|
|
9
9
|
|
|
10
10
|
## Usage
|
|
11
11
|
|
|
12
|
+
```ruby
|
|
13
|
+
index_url = MassiveSitemap.generate(:url => 'test.de/') do
|
|
14
|
+
add "dummy"
|
|
15
|
+
end
|
|
16
|
+
MassiveSitemap.ping(index_url)
|
|
17
|
+
```
|
|
18
|
+
|
|
12
19
|
* clear structure
|
|
13
20
|
* allows extension (S3)
|
|
14
21
|
|
data/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
2.0.0.
|
|
1
|
+
2.0.0.rc3
|
|
@@ -3,22 +3,20 @@ module MassiveSitemap
|
|
|
3
3
|
|
|
4
4
|
class Base
|
|
5
5
|
OPTS = {
|
|
6
|
-
:
|
|
6
|
+
:url => nil,
|
|
7
7
|
:indent_by => 2
|
|
8
8
|
}
|
|
9
9
|
|
|
10
|
-
HEADER_NAME
|
|
10
|
+
HEADER_NAME = 'urlset'
|
|
11
11
|
HEADER_ATTRIBUTES = {
|
|
12
|
-
'xmlns'
|
|
13
|
-
'xmlns:xsi'
|
|
12
|
+
'xmlns' => 'http://www.sitemaps.org/schemas/sitemap/0.9',
|
|
13
|
+
'xmlns:xsi' => "http://www.w3.org/2001/XMLSchema-instance",
|
|
14
14
|
'xsi:schemaLocation' => "http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd"
|
|
15
15
|
}
|
|
16
16
|
|
|
17
|
-
attr_reader :options
|
|
18
|
-
|
|
19
17
|
def initialize(writer, options = {}, &block)
|
|
20
18
|
@writer = writer
|
|
21
|
-
@options = OPTS.merge(options)
|
|
19
|
+
@options = self.class::OPTS.merge(options)
|
|
22
20
|
@opened_tags = []
|
|
23
21
|
|
|
24
22
|
if block
|
|
@@ -32,8 +30,9 @@ module MassiveSitemap
|
|
|
32
30
|
end
|
|
33
31
|
|
|
34
32
|
def add(path, attrs = {})
|
|
35
|
-
add_url! File.join(
|
|
33
|
+
add_url! ::File.join(url, path), attrs
|
|
36
34
|
rescue MassiveSitemap::Writer::File::FileExistsException => e
|
|
35
|
+
# don't fail here
|
|
37
36
|
end
|
|
38
37
|
|
|
39
38
|
def init!(&block)
|
|
@@ -45,7 +44,7 @@ module MassiveSitemap
|
|
|
45
44
|
|
|
46
45
|
def close!(indent = true)
|
|
47
46
|
if name = @opened_tags.pop
|
|
48
|
-
@writer.print "\n" + ' ' * options[:indent_by] * @opened_tags.size if indent
|
|
47
|
+
@writer.print "\n" + ' ' * @options[:indent_by] * @opened_tags.size if indent
|
|
49
48
|
@writer.print "</#{name}>"
|
|
50
49
|
if @opened_tags.size == 0
|
|
51
50
|
@writer.close!
|
|
@@ -87,14 +86,14 @@ module MassiveSitemap
|
|
|
87
86
|
|
|
88
87
|
def open!(name, attrs = {})
|
|
89
88
|
attrs = attrs.map { |attr, value| %Q( #{attr}="#{value}") }.join('')
|
|
90
|
-
@writer.print "\n" + ' ' * options[:indent_by] * @opened_tags.size
|
|
89
|
+
@writer.print "\n" + ' ' * @options[:indent_by] * @opened_tags.size
|
|
91
90
|
@opened_tags << name
|
|
92
91
|
@writer.print "<#{name}#{attrs}>"
|
|
93
92
|
end
|
|
94
93
|
|
|
95
94
|
private
|
|
96
|
-
def
|
|
97
|
-
schema, host = @options[:
|
|
95
|
+
def url
|
|
96
|
+
schema, host = @options[:url].scan(/^(https?:\/\/)?(.+?)\/?$/).flatten
|
|
98
97
|
"#{schema || 'http://'}#{host}/"
|
|
99
98
|
rescue
|
|
100
99
|
""
|
|
@@ -14,6 +14,15 @@ module MassiveSitemap
|
|
|
14
14
|
add path, :last_modified => last_modified
|
|
15
15
|
end
|
|
16
16
|
end
|
|
17
|
+
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def self.generate(writer, options = {}, &block)
|
|
21
|
+
index_url(super, writer)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def self.index_url(builder, writer)
|
|
25
|
+
::File.join(builder.send(:url), writer.options[:filename])
|
|
17
26
|
end
|
|
18
27
|
|
|
19
28
|
def add_url!(location, attrs = {})
|
|
@@ -6,12 +6,15 @@ module MassiveSitemap
|
|
|
6
6
|
class Rotating < Base
|
|
7
7
|
NUM_URLS = 1..50_000
|
|
8
8
|
|
|
9
|
+
OPTS = Base::OPTS.merge(
|
|
10
|
+
:max_per_sitemap => NUM_URLS.max
|
|
11
|
+
)
|
|
12
|
+
|
|
9
13
|
def initialize(writer, options = {}, &block)
|
|
10
|
-
@max_urls = options[:max_per_sitemap] || NUM_URLS.max
|
|
11
14
|
@rotations = 0
|
|
12
|
-
@urls
|
|
15
|
+
@urls = 0
|
|
13
16
|
|
|
14
|
-
|
|
17
|
+
if options[:max_per_sitemap] && !NUM_URLS.member?(options[:max_per_sitemap])
|
|
15
18
|
raise ArgumentError, %Q(":max_per_sitemap" must be greater than #{NUM_URLS.min} and smaller than #{NUM_URLS.max})
|
|
16
19
|
end
|
|
17
20
|
|
|
@@ -22,7 +25,7 @@ module MassiveSitemap
|
|
|
22
25
|
# with same file name but -<counter> appendend
|
|
23
26
|
def init!(&block)
|
|
24
27
|
unless @writer.inited?
|
|
25
|
-
@urls
|
|
28
|
+
@urls = 0
|
|
26
29
|
filename = filename_with_rotation(@writer.options[:filename], @rotations)
|
|
27
30
|
@rotations += 1
|
|
28
31
|
@writer.init! :filename => filename
|
|
@@ -31,7 +34,7 @@ module MassiveSitemap
|
|
|
31
34
|
end
|
|
32
35
|
|
|
33
36
|
def add_url!(location, attrs = {})
|
|
34
|
-
if @urls >= @
|
|
37
|
+
if @urls >= @options[:max_per_sitemap]
|
|
35
38
|
close!
|
|
36
39
|
end
|
|
37
40
|
super
|
data/lib/massive_sitemap/ping.rb
CHANGED
|
@@ -1,21 +1,29 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
PING = {
|
|
4
|
-
:google => 'http://www.google.comwebmasters/tools/ping?sitemap=%s';
|
|
5
|
-
:bing => 'http://www.bing.com/webmaster/ping.aspx?siteMap=%s',
|
|
6
|
-
:ask => 'http://submissions.ask.com/ping?sitemap=%s'
|
|
7
|
-
}
|
|
1
|
+
require 'cgi'
|
|
2
|
+
require 'open-uri'
|
|
8
3
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
4
|
+
module MassiveSitemap
|
|
5
|
+
ENGINES_URLS = {
|
|
6
|
+
:google => 'http://www.google.com/webmasters/tools/ping?sitemap=%s',
|
|
7
|
+
:bing => 'http://www.bing.com/webmaster/ping.aspx?siteMap=%s',
|
|
8
|
+
:ask => 'http://submissions.ask.com/ping?sitemap=%s',
|
|
9
|
+
}
|
|
13
10
|
|
|
14
|
-
|
|
11
|
+
def ping(url, engines = ENGINES_URLS.keys)
|
|
12
|
+
url = verify_and_escape(url)
|
|
15
13
|
|
|
16
|
-
|
|
17
|
-
|
|
14
|
+
Array(engines).each do |engine|
|
|
15
|
+
if engine_url = ENGINES_URLS[engine]
|
|
16
|
+
open(engine_url % url)
|
|
18
17
|
end
|
|
19
18
|
end
|
|
20
19
|
end
|
|
20
|
+
module_function :ping
|
|
21
|
+
|
|
22
|
+
private
|
|
23
|
+
def verify_and_escape(url)
|
|
24
|
+
schema, host, path = url.scan(/^(https?:\/\/)?(.+?)(\/.+)$/).flatten
|
|
25
|
+
raise URI::InvalidURIError, url if path.to_s.empty?
|
|
26
|
+
CGI::escape("#{schema || 'http://'}#{host}#{path}")
|
|
27
|
+
end
|
|
28
|
+
module_function :verify_and_escape
|
|
21
29
|
end
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
require 'fileutils'
|
|
2
|
-
require
|
|
2
|
+
require 'massive_sitemap/writer/base'
|
|
3
3
|
|
|
4
4
|
# Write into File
|
|
5
5
|
|
|
@@ -10,15 +10,16 @@ module MassiveSitemap
|
|
|
10
10
|
class FileExistsException < IOError; end
|
|
11
11
|
|
|
12
12
|
OPTS = Base::OPTS.merge(
|
|
13
|
-
:
|
|
13
|
+
:root => '.',
|
|
14
14
|
:force_overwrite => false,
|
|
15
15
|
:filename => "sitemap.xml",
|
|
16
16
|
:index_filename => "sitemap_index.xml",
|
|
17
17
|
)
|
|
18
18
|
|
|
19
19
|
def open_stream
|
|
20
|
-
|
|
21
|
-
|
|
20
|
+
::File.dirname(tmp_filename).tap do |dir|
|
|
21
|
+
FileUtils.mkdir_p(dir) unless ::File.exists?(dir)
|
|
22
|
+
end
|
|
22
23
|
::File.open(tmp_filename, 'w:ASCII-8BIT')
|
|
23
24
|
end
|
|
24
25
|
|
|
@@ -45,7 +46,7 @@ module MassiveSitemap
|
|
|
45
46
|
|
|
46
47
|
private
|
|
47
48
|
def filename
|
|
48
|
-
::File.join options[:
|
|
49
|
+
::File.join options[:root], options[:filename]
|
|
49
50
|
end
|
|
50
51
|
|
|
51
52
|
def tmp_filename
|
|
@@ -53,7 +54,7 @@ module MassiveSitemap
|
|
|
53
54
|
end
|
|
54
55
|
|
|
55
56
|
def files
|
|
56
|
-
Dir[::File.join(options[:
|
|
57
|
+
Dir[::File.join(options[:root], "*.xml")]
|
|
57
58
|
end
|
|
58
59
|
end
|
|
59
60
|
|
data/lib/massive_sitemap.rb
CHANGED
|
@@ -2,51 +2,51 @@ require 'massive_sitemap/writer/file'
|
|
|
2
2
|
require 'massive_sitemap/writer/gzip_file'
|
|
3
3
|
require 'massive_sitemap/builder/rotating'
|
|
4
4
|
require 'massive_sitemap/builder/index'
|
|
5
|
+
require 'massive_sitemap/ping'
|
|
5
6
|
|
|
6
|
-
# Page at -> <
|
|
7
|
+
# Page at -> <url>
|
|
7
8
|
# http://example.de/dir/
|
|
8
9
|
|
|
9
|
-
# Index at
|
|
10
|
+
# Index at -> <index_url>
|
|
10
11
|
# http://sitemap.example.de/index-dir/
|
|
11
12
|
|
|
12
|
-
# Save at -> <
|
|
13
|
+
# Save at -> <root>
|
|
13
14
|
# /root/dir/ -> <document_root>/<document_path>
|
|
14
15
|
|
|
15
16
|
module MassiveSitemap
|
|
16
17
|
DEFAULTS = {
|
|
17
18
|
# global
|
|
18
|
-
:
|
|
19
|
+
:index_url => nil,
|
|
19
20
|
:gzip => false,
|
|
20
21
|
:writer => MassiveSitemap::Writer::File,
|
|
21
22
|
|
|
22
23
|
# writer
|
|
23
|
-
:
|
|
24
|
+
:root => '.',
|
|
24
25
|
:force_overwrite => false,
|
|
25
26
|
:filename => "sitemap.xml",
|
|
26
27
|
:index_filename => "sitemap_index.xml",
|
|
27
28
|
|
|
28
29
|
# builder
|
|
29
|
-
:
|
|
30
|
+
:url => nil,
|
|
30
31
|
:indent_by => 2,
|
|
31
32
|
}
|
|
32
33
|
|
|
33
34
|
def generate(options = {}, &block)
|
|
34
35
|
@options = DEFAULTS.merge options
|
|
35
36
|
|
|
36
|
-
unless @options[:
|
|
37
|
-
raise ArgumentError, 'you must specify ":
|
|
37
|
+
unless @options[:url]
|
|
38
|
+
raise ArgumentError, 'you must specify ":url" string'
|
|
38
39
|
end
|
|
39
|
-
@options[:
|
|
40
|
+
@options[:index_url] ||= @options[:url]
|
|
40
41
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
@options[:writer] = MassiveSitemap::Writer::GzipFile if @options[:gzip]
|
|
42
|
+
@options[:writer] = Writer::GzipFile if @options[:gzip]
|
|
44
43
|
|
|
45
44
|
@writer = @options[:writer].new @options
|
|
46
45
|
Builder::Rotating.generate(@writer, @options, &block)
|
|
47
46
|
|
|
48
|
-
@writer.
|
|
49
|
-
Builder::Index.generate(@writer, @options.merge(:
|
|
47
|
+
@writer.init!(:filename => @options[:index_filename], :force_overwrite => true)
|
|
48
|
+
Builder::Index.generate(@writer, @options.merge(:url => @options[:index_url]))
|
|
50
49
|
end
|
|
51
50
|
module_function :generate
|
|
51
|
+
|
|
52
52
|
end
|
data/spec/builder/base_spec.rb
CHANGED
|
@@ -129,7 +129,7 @@ describe MassiveSitemap::Builder::Base do
|
|
|
129
129
|
end
|
|
130
130
|
end
|
|
131
131
|
|
|
132
|
-
describe ".
|
|
132
|
+
describe ".url" do
|
|
133
133
|
URLS = %w(
|
|
134
134
|
http://test.de/
|
|
135
135
|
test.de/
|
|
@@ -138,12 +138,12 @@ describe MassiveSitemap::Builder::Base do
|
|
|
138
138
|
|
|
139
139
|
URLS.each do |url|
|
|
140
140
|
it "transforms to valid url" do
|
|
141
|
-
MassiveSitemap::Builder.new(writer, :
|
|
141
|
+
MassiveSitemap::Builder.new(writer, :url => url).send(:url).should == "http://test.de/"
|
|
142
142
|
end
|
|
143
143
|
end
|
|
144
144
|
|
|
145
145
|
it "transforms to valid url with https" do
|
|
146
|
-
MassiveSitemap::Builder.new(writer, :
|
|
146
|
+
MassiveSitemap::Builder.new(writer, :url => "https://test.de/").send(:url).should == "https://test.de/"
|
|
147
147
|
end
|
|
148
148
|
end
|
|
149
149
|
end
|
data/spec/builder/index_spec.rb
CHANGED
|
@@ -18,9 +18,13 @@ describe MassiveSitemap::Builder::Index do
|
|
|
18
18
|
writer.should == %Q(#{INDEX_HEADER}\n<loc>/test</loc>\n</sitemap>\n</sitemapindex>)
|
|
19
19
|
end
|
|
20
20
|
|
|
21
|
-
it 'include
|
|
22
|
-
MassiveSitemap::Builder::Index.new(writer, :
|
|
21
|
+
it 'include url' do
|
|
22
|
+
MassiveSitemap::Builder::Index.new(writer, :url => "test.de", :indent_by => 0)
|
|
23
23
|
writer.should include("<loc>http://test.de/test</loc>")
|
|
24
24
|
end
|
|
25
25
|
|
|
26
|
+
it 'returns index_url' do
|
|
27
|
+
writer = MassiveSitemap::Writer::File.new(:filename => "sitemap_index.xml")
|
|
28
|
+
MassiveSitemap::Builder::Index.generate(writer, :url => "test.de").should == "http://test.de/sitemap_index.xml"
|
|
29
|
+
end
|
|
26
30
|
end
|
|
@@ -21,138 +21,160 @@ describe MassiveSitemap do
|
|
|
21
21
|
FileUtils.rm(filename2) rescue nil
|
|
22
22
|
end
|
|
23
23
|
|
|
24
|
-
describe "
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
MassiveSitemap.generate
|
|
28
|
-
end.to raise_error(ArgumentError)
|
|
29
|
-
end
|
|
30
|
-
|
|
31
|
-
it "does not create empty sitemap file" do
|
|
32
|
-
expect do
|
|
33
|
-
MassiveSitemap.generate(:base_url => 'test.de/')
|
|
34
|
-
end.to_not change { ::File.exists?(filename) }
|
|
35
|
-
end
|
|
36
|
-
|
|
37
|
-
context "custom writer" do
|
|
38
|
-
after do
|
|
39
|
-
FileUtils.rm(gz_filename(index_filename)) rescue nil
|
|
40
|
-
FileUtils.rm(gz_filename) rescue nil
|
|
41
|
-
end
|
|
42
|
-
|
|
43
|
-
it 'takes gzips writer' do
|
|
24
|
+
describe "generate" do
|
|
25
|
+
context "initalize" do
|
|
26
|
+
it 'fail if no url given' do
|
|
44
27
|
expect do
|
|
45
|
-
MassiveSitemap.generate
|
|
46
|
-
|
|
47
|
-
end
|
|
48
|
-
end.to change { ::File.exists?(gz_filename) }.to(true)
|
|
28
|
+
MassiveSitemap.generate
|
|
29
|
+
end.to raise_error(ArgumentError)
|
|
49
30
|
end
|
|
50
31
|
|
|
51
|
-
it
|
|
32
|
+
it "does not create empty sitemap file" do
|
|
52
33
|
expect do
|
|
53
|
-
MassiveSitemap.generate(:
|
|
54
|
-
|
|
55
|
-
end
|
|
56
|
-
end.to change { ::File.exists?(gz_filename) }.to(true)
|
|
34
|
+
MassiveSitemap.generate(:url => 'test.de/')
|
|
35
|
+
end.to_not change { ::File.exists?(filename) }
|
|
57
36
|
end
|
|
58
|
-
end
|
|
59
|
-
end
|
|
60
37
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
38
|
+
context "custom writer" do
|
|
39
|
+
after do
|
|
40
|
+
FileUtils.rm(gz_filename(index_filename)) rescue nil
|
|
41
|
+
FileUtils.rm(gz_filename) rescue nil
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
it 'takes gzips writer' do
|
|
45
|
+
expect do
|
|
46
|
+
MassiveSitemap.generate(:url => 'test.de/', :gzip => true) do
|
|
47
|
+
add "dummy"
|
|
48
|
+
end
|
|
49
|
+
end.to change { ::File.exists?(gz_filename) }.to(true)
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
it 'takes custom writer' do
|
|
53
|
+
expect do
|
|
54
|
+
MassiveSitemap.generate(:url => 'test.de/', :writer => MassiveSitemap::Writer::GzipFile) do
|
|
55
|
+
add "dummy"
|
|
56
|
+
end
|
|
57
|
+
end.to change { ::File.exists?(gz_filename) }.to(true)
|
|
58
|
+
end
|
|
65
59
|
end
|
|
66
|
-
output.should include("<loc>http://test.de/track/name</loc>")
|
|
67
60
|
end
|
|
68
61
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
62
|
+
context "generate" do
|
|
63
|
+
it 'adds url' do
|
|
64
|
+
MassiveSitemap.generate(:url => 'test.de') do
|
|
65
|
+
add "track/name"
|
|
66
|
+
end
|
|
67
|
+
output.should include("<loc>http://test.de/track/name</loc>")
|
|
72
68
|
end
|
|
73
|
-
output.should include("<loc>http://test.de/track/name</loc>")
|
|
74
|
-
end
|
|
75
69
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
expect do
|
|
79
|
-
MassiveSitemap.generate(:base_url => 'test.de/') do
|
|
70
|
+
it 'adds url with root slash' do
|
|
71
|
+
MassiveSitemap.generate(:url => 'test.de/') do
|
|
80
72
|
add "/track/name"
|
|
81
73
|
end
|
|
82
|
-
|
|
83
|
-
|
|
74
|
+
output.should include("<loc>http://test.de/track/name</loc>")
|
|
75
|
+
end
|
|
84
76
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
add "/set/name"
|
|
77
|
+
it "doesn't fail for existing file" do
|
|
78
|
+
File.open(filename, 'w') {}
|
|
79
|
+
expect do
|
|
80
|
+
MassiveSitemap.generate(:url => 'test.de/') do
|
|
81
|
+
add "/track/name"
|
|
91
82
|
end
|
|
92
|
-
end
|
|
93
|
-
output(filename2).should include("<loc>http://test.de/set/name</loc>")
|
|
83
|
+
end.to_not change { File.stat(filename).mtime }
|
|
94
84
|
end
|
|
95
85
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
86
|
+
context 'nested generation' do
|
|
87
|
+
it 'adds url of nested builder' do
|
|
88
|
+
MassiveSitemap.generate(:url => 'test.de/') do
|
|
89
|
+
writer = @writer.class.new(@options.merge(:filename => 'sitemap2.xml'))
|
|
90
|
+
MassiveSitemap::Builder::Rotating.new(writer, @options) do
|
|
91
|
+
add "/set/name"
|
|
92
|
+
end
|
|
102
93
|
end
|
|
94
|
+
output(filename2).should include("<loc>http://test.de/set/name</loc>")
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
it 'executes block altough first sitemap exists' do
|
|
98
|
+
File.open(filename, 'w') {}
|
|
99
|
+
MassiveSitemap.generate(:url => 'test.de/') do
|
|
100
|
+
writer = @writer.class.new(@options.merge(:filename => 'sitemap2.xml'))
|
|
101
|
+
MassiveSitemap::Builder::Rotating.new(writer, @options) do
|
|
102
|
+
add "/set/name"
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
output(filename2).should include("<loc>http://test.de/set/name</loc>")
|
|
103
106
|
end
|
|
104
|
-
output(filename2).should include("<loc>http://test.de/set/name</loc>")
|
|
105
107
|
end
|
|
108
|
+
|
|
106
109
|
end
|
|
107
110
|
|
|
108
|
-
|
|
111
|
+
context "generate_index" do
|
|
112
|
+
let(:lastmod) { File.stat(index_filename).mtime.utc.strftime('%Y-%m-%dT%H:%M:%S+00:00') }
|
|
109
113
|
|
|
110
|
-
|
|
111
|
-
|
|
114
|
+
it "does not create empty files" do
|
|
115
|
+
MassiveSitemap.generate(:url => 'test.de/')
|
|
116
|
+
::File.exists?(index_filename).should be_false
|
|
117
|
+
end
|
|
112
118
|
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
119
|
+
it 'includes urls' do
|
|
120
|
+
MassiveSitemap.generate(:url => 'test.de/', :indent_by => 0) do
|
|
121
|
+
add "dummy"
|
|
122
|
+
end
|
|
117
123
|
|
|
118
|
-
|
|
119
|
-
MassiveSitemap.generate(:base_url => 'test.de/', :indent_by => 0) do
|
|
120
|
-
add "dummy"
|
|
124
|
+
output(index_filename).should include("<sitemap>\n<loc>http://test.de/sitemap.xml</loc>\n<lastmod>#{lastmod}</lastmod>\n</sitemap>")
|
|
121
125
|
end
|
|
122
126
|
|
|
123
|
-
|
|
124
|
-
|
|
127
|
+
it 'includes index base url' do
|
|
128
|
+
MassiveSitemap.generate(:url => 'test.de/', :index_url => 'index.de/') do
|
|
129
|
+
add "dummy"
|
|
130
|
+
end
|
|
125
131
|
|
|
126
|
-
|
|
127
|
-
MassiveSitemap.generate(:base_url => 'test.de/', :index_base_url => 'index.de/') do
|
|
128
|
-
add "dummy"
|
|
132
|
+
output(index_filename).should include("<loc>http://index.de/sitemap.xml</loc>")
|
|
129
133
|
end
|
|
130
134
|
|
|
131
|
-
|
|
132
|
-
|
|
135
|
+
it 'overwrites existing one' do
|
|
136
|
+
File.open(index_filename, 'w') {}
|
|
137
|
+
MassiveSitemap.generate(:url => 'test.de/', :index_url => 'index.de/') do
|
|
138
|
+
add "dummy"
|
|
139
|
+
end
|
|
133
140
|
|
|
134
|
-
|
|
135
|
-
File.open(index_filename, 'w') {}
|
|
136
|
-
MassiveSitemap.generate(:base_url => 'test.de/', :index_base_url => 'index.de/') do
|
|
137
|
-
add "dummy"
|
|
141
|
+
output(index_filename).should include("<loc>http://index.de/sitemap.xml</loc>")
|
|
138
142
|
end
|
|
139
143
|
|
|
140
|
-
|
|
141
|
-
|
|
144
|
+
context "gziped" do
|
|
145
|
+
after do
|
|
146
|
+
FileUtils.rm(gz_filename(index_filename)) rescue nil
|
|
147
|
+
FileUtils.rm(gz_filename) rescue nil
|
|
148
|
+
end
|
|
142
149
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
150
|
+
it 'creates sitemap file' do
|
|
151
|
+
expect do
|
|
152
|
+
MassiveSitemap.generate(:url => 'test.de/', :writer => MassiveSitemap::Writer::GzipFile) do
|
|
153
|
+
add "dummy"
|
|
154
|
+
end
|
|
155
|
+
end.to change { ::File.exists?(gz_filename(index_filename)) }.to(true)
|
|
156
|
+
end
|
|
147
157
|
end
|
|
158
|
+
end
|
|
159
|
+
end
|
|
148
160
|
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
161
|
+
describe "ping" do
|
|
162
|
+
PINGS = [
|
|
163
|
+
"http://www.google.com/webmasters/tools/ping?sitemap=http%3A%2F%2Ftest.de%2Fsitemap_index.xml",
|
|
164
|
+
"http://www.bing.com/webmaster/ping.aspx?siteMap=http%3A%2F%2Ftest.de%2Fsitemap_index.xml",
|
|
165
|
+
"http://submissions.ask.com/ping?sitemap=http%3A%2F%2Ftest.de%2Fsitemap_index.xml",
|
|
166
|
+
]
|
|
167
|
+
|
|
168
|
+
it 'calls all engines' do
|
|
169
|
+
MassiveSitemap.should_receive(:open).exactly(3).times.with { |arg|
|
|
170
|
+
arg.should == PINGS.shift
|
|
171
|
+
}.and_return(true)
|
|
172
|
+
|
|
173
|
+
index_url = MassiveSitemap.generate(:url => 'test.de/') do
|
|
174
|
+
add "dummy"
|
|
155
175
|
end
|
|
176
|
+
MassiveSitemap.ping(index_url)
|
|
156
177
|
end
|
|
157
178
|
end
|
|
179
|
+
|
|
158
180
|
end
|
data/spec/ping_spec.rb
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
require "spec_helper"
|
|
2
|
+
|
|
3
|
+
describe MassiveSitemap do
|
|
4
|
+
describe ".ping" do
|
|
5
|
+
let(:url) { "http://www.example.com" }
|
|
6
|
+
|
|
7
|
+
describe "verify_and_escape" do
|
|
8
|
+
it { MassiveSitemap.verify_and_escape("example.com/test").should == "http%3A%2F%2Fexample.com%2Ftest" }
|
|
9
|
+
it { MassiveSitemap.verify_and_escape("http://example.com/test").should == "http%3A%2F%2Fexample.com%2Ftest" }
|
|
10
|
+
it { MassiveSitemap.verify_and_escape("https://example.com/test").should == "https%3A%2F%2Fexample.com%2Ftest" }
|
|
11
|
+
|
|
12
|
+
it "raise if invalid url" do
|
|
13
|
+
expect do
|
|
14
|
+
MassiveSitemap.verify_and_escape("example.com/")
|
|
15
|
+
end.to raise_error URI::InvalidURIError
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
describe "ping" do
|
|
20
|
+
it "calles google and ask" do
|
|
21
|
+
MassiveSitemap.should_receive(:open).twice()
|
|
22
|
+
MassiveSitemap.ping(url, [:google, :ask])
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
it "doesn't fail for unknown engines" do
|
|
26
|
+
expect do
|
|
27
|
+
MassiveSitemap.ping(url, :unknown)
|
|
28
|
+
end.to_not raise_error
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
data/spec/writer/file_spec.rb
CHANGED
|
@@ -12,33 +12,38 @@ describe MassiveSitemap::Writer::File do
|
|
|
12
12
|
FileUtils.rm(filename2) rescue nil
|
|
13
13
|
end
|
|
14
14
|
|
|
15
|
-
describe "
|
|
16
|
-
let(:
|
|
15
|
+
describe "root" do
|
|
16
|
+
let(:root) { "test/test2" }
|
|
17
17
|
|
|
18
|
-
|
|
19
|
-
|
|
18
|
+
after do
|
|
19
|
+
FileUtils.rm_rf(root) rescue nil
|
|
20
20
|
end
|
|
21
21
|
|
|
22
|
-
|
|
23
|
-
|
|
22
|
+
it 'mkdir_p folder' do
|
|
23
|
+
expect do
|
|
24
|
+
MassiveSitemap::Writer::File.new(:root => root).tap do |w|
|
|
25
|
+
w.init!
|
|
26
|
+
w.close!
|
|
27
|
+
end
|
|
28
|
+
end.to change { File.exists?(root) }.to(true)
|
|
24
29
|
end
|
|
25
30
|
|
|
26
|
-
it 'appends
|
|
31
|
+
it 'appends root' do
|
|
27
32
|
expect do
|
|
28
|
-
MassiveSitemap::Writer::File.new(:
|
|
33
|
+
MassiveSitemap::Writer::File.new(:root => root).tap do |w|
|
|
29
34
|
w.init!
|
|
30
35
|
w.close!
|
|
31
36
|
end
|
|
32
|
-
end.to change { File.exists?("
|
|
37
|
+
end.to change { File.exists?("#{root}/#{filename}") }.to(true)
|
|
33
38
|
end
|
|
34
39
|
|
|
35
|
-
it 'appends
|
|
40
|
+
it 'appends root' do
|
|
36
41
|
expect do
|
|
37
|
-
MassiveSitemap::Writer::File.new(:
|
|
42
|
+
MassiveSitemap::Writer::File.new(:root => "#{root}/").tap do |w|
|
|
38
43
|
w.init!
|
|
39
44
|
w.close!
|
|
40
45
|
end
|
|
41
|
-
end.to change { File.exists?("
|
|
46
|
+
end.to change { File.exists?("#{root}/#{filename}") }.to(true)
|
|
42
47
|
end
|
|
43
48
|
end
|
|
44
49
|
|
|
@@ -17,13 +17,13 @@ describe MassiveSitemap::Writer::GzipFile do
|
|
|
17
17
|
end.to change { File.exists?(gz_filename) }.to(true)
|
|
18
18
|
end
|
|
19
19
|
|
|
20
|
-
context "with
|
|
21
|
-
let(:
|
|
20
|
+
context "with root" do
|
|
21
|
+
let(:root) { "sitemap"}
|
|
22
22
|
|
|
23
|
-
let(:writer) { MassiveSitemap::Writer::GzipFile.new(:
|
|
23
|
+
let(:writer) { MassiveSitemap::Writer::GzipFile.new(:root => root).tap { |w| w.init! } }
|
|
24
24
|
|
|
25
25
|
after do
|
|
26
|
-
FileUtils.rm_rf(
|
|
26
|
+
FileUtils.rm_rf(root) rescue nil
|
|
27
27
|
end
|
|
28
28
|
|
|
29
29
|
it 'creates gzip file in document root' do
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: massive_sitemap
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2.0.0.
|
|
4
|
+
version: 2.0.0.rc3
|
|
5
5
|
prerelease: 6
|
|
6
6
|
platform: ruby
|
|
7
7
|
authors:
|
|
@@ -13,7 +13,7 @@ date: 2012-02-12 00:00:00.000000000Z
|
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: rake
|
|
16
|
-
requirement: &
|
|
16
|
+
requirement: &70334535721800 !ruby/object:Gem::Requirement
|
|
17
17
|
none: false
|
|
18
18
|
requirements:
|
|
19
19
|
- - ! '>='
|
|
@@ -21,10 +21,10 @@ dependencies:
|
|
|
21
21
|
version: '0'
|
|
22
22
|
type: :development
|
|
23
23
|
prerelease: false
|
|
24
|
-
version_requirements: *
|
|
24
|
+
version_requirements: *70334535721800
|
|
25
25
|
- !ruby/object:Gem::Dependency
|
|
26
26
|
name: rspec
|
|
27
|
-
requirement: &
|
|
27
|
+
requirement: &70334535721340 !ruby/object:Gem::Requirement
|
|
28
28
|
none: false
|
|
29
29
|
requirements:
|
|
30
30
|
- - ! '>='
|
|
@@ -32,7 +32,7 @@ dependencies:
|
|
|
32
32
|
version: '0'
|
|
33
33
|
type: :development
|
|
34
34
|
prerelease: false
|
|
35
|
-
version_requirements: *
|
|
35
|
+
version_requirements: *70334535721340
|
|
36
36
|
description: MassiveSitemap - build huge sitemaps painfree. Differential updates keeps
|
|
37
37
|
generation time short and reduces load on DB. It's heavealy inspired by BigSitemaps
|
|
38
38
|
and offers compatiable API
|
|
@@ -66,6 +66,7 @@ files:
|
|
|
66
66
|
- spec/builder/index_spec.rb
|
|
67
67
|
- spec/builder/rotating_spec.rb
|
|
68
68
|
- spec/massive_sitemap_spec.rb
|
|
69
|
+
- spec/ping_spec.rb
|
|
69
70
|
- spec/spec_helper.rb
|
|
70
71
|
- spec/writer/file_spec.rb
|
|
71
72
|
- spec/writer/gzip_file_spec.rb
|
|
@@ -99,6 +100,7 @@ test_files:
|
|
|
99
100
|
- spec/builder/index_spec.rb
|
|
100
101
|
- spec/builder/rotating_spec.rb
|
|
101
102
|
- spec/massive_sitemap_spec.rb
|
|
103
|
+
- spec/ping_spec.rb
|
|
102
104
|
- spec/spec_helper.rb
|
|
103
105
|
- spec/writer/file_spec.rb
|
|
104
106
|
- spec/writer/gzip_file_spec.rb
|