massive_sitemap 2.0.0.rc1

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg
+.rvmrc

data/.travis.yml

@@ -0,0 +1,3 @@
+rvm:
+  - 1.9.2
+script: "bundle exec rake spec"

data/CHANGELOG.md

@@ -0,0 +1,28 @@
+# Changes
+
+## vx.x.x - ???
+
+## v2.0.x - ???
+
+  * updated/fixed Ping
+  * updated Docu
+  * switch to writer chain
+  * add BigSitemap API
+
+## v2.0.0 - 13-02-2012
+  _inital release_
+
+  * restructured gem completely based on BigSitemap gem
+  * seperated logic in two major parts:
+    * Builder -> creates content
+    * Writer -> stores content
+  * added several implementations/specifiaction of builder/writer
+  * added generator for default setup
+  * added specs
+  * writer overwrite detection
+  * added Index generation
+  * don't init new writer all the time
+  * move inited status to writer
+  * move index build into indexer and resource handling/selection into writer
+  * manifest handling:
+  * moved Amazon S3 integration to [massive_sitemap-writer-s3](https://github.com/rngtng/massive_sitemap-writer-s3)

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in massive_sitemap.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,24 @@
+PATH
+  remote: .
+  specs:
+    massive_sitemap (0.0.1)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    diff-lcs (1.1.3)
+    rspec (2.8.0)
+      rspec-core (~> 2.8.0)
+      rspec-expectations (~> 2.8.0)
+      rspec-mocks (~> 2.8.0)
+    rspec-core (2.8.0)
+    rspec-expectations (2.8.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.8.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  massive_sitemap!
+  rspec

data/README.md

@@ -0,0 +1,15 @@
+# MassiveSitemap
+
+[![](http://travis-ci.org/rngtng/massive_sitemap.png)](http://travis-ci.org/rngtng/massive_sitemap)
+
+Build painfree sitemaps for websites with millions of pages
+
+MassiveSitemap is a successor project of [BigSitemap](https://github.com/alexrabarts/big_sitemap), a [Sitemap](http://sitemaps.org) generator for websites with millions of pages.
+It implements various generation stategies, e.g. to split large Sitemaps into multiple files, gzip files to minimize bandwidth usage, or incremental updates. It offers API is very similar to _BigSitemap_ and therefor can be set up with just a few lines of code and is compatible with just about any framework.
+
+## Usage
+
+* clear structure
+* allows extension (S3)
+
+MassiveSitemap - build huge sitemaps painfree. Differential updates keeps generation time short and reduces load on DB. It's heavealy inspired by BigSitemaps and offers compatiable API

data/Rakefile

@@ -0,0 +1,8 @@
+require 'bundler/gem_tasks'
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.pattern = "./spec/**/*_spec.rb" # don't need this, it's default.
+end
+
+task :default => :spec

data/VERSION

@@ -0,0 +1 @@
+2.0.0.rc1

data/lib/massive_sitemap/builder/base.rb

@@ -0,0 +1,104 @@
+module MassiveSitemap
+  module Builder
+
+    class Base
+      OPTS = {
+        :base_url  => nil,
+        :indent_by => 2
+      }
+
+      HEADER_NAME = 'urlset'
+      HEADER_ATTRIBUTES = {
+        'xmlns' => 'http://www.sitemaps.org/schemas/sitemap/0.9',
+        'xmlns:xsi' => "http://www.w3.org/2001/XMLSchema-instance",
+        'xsi:schemaLocation' => "http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd"
+      }
+
+      attr_reader :options
+
+      def initialize(writer, options = {}, &block)
+        @writer      = writer
+        @options     = OPTS.merge(options)
+        @opened_tags = []
+
+        if block
+          instance_eval(&block)
+          close!
+        end
+      end
+
+      def self.generate(writer, options = {}, &block)
+        self.new(writer, options, &block)
+      end
+
+      def add(path, attrs = {})
+        add_url! File.join(base_url, path), attrs
+      rescue MassiveSitemap::Writer::File::FileExistsException => e
+      end
+
+      def init!(&block)
+        unless @writer.inited?
+          @writer.init!
+          header!(&block)
+        end
+      end
+
+      def close!(indent = true)
+        if name = @opened_tags.pop
+          @writer.print "\n" + ' ' * options[:indent_by] * @opened_tags.size if indent
+          @writer.print "</#{name}>"
+          if @opened_tags.size == 0
+            @writer.close!
+            true
+          end
+        end
+      end
+
+      private
+      def header!(&block)
+        @writer.print '<?xml version="1.0" encoding="UTF-8"?>'
+        tag! self.class::HEADER_NAME, self.class::HEADER_ATTRIBUTES, &block
+      end
+
+      def add_url!(location, attrs = {})
+        init!
+
+        tag! 'url' do
+          tag! 'loc', location
+          tag! 'lastmod', attrs[:last_modified].utc.strftime('%Y-%m-%dT%H:%M:%S+00:00') if attrs[:last_modified]
+          tag! 'changefreq', attrs[:change_frequency] if attrs[:change_frequency]
+          tag! 'priority', attrs[:priority].to_s if attrs[:priority]
+        end
+      end
+
+      def tag!(name, content = nil, attrs = {}, &block)
+        attrs = content if content.is_a? Hash
+        open!(name, attrs)
+        if content.is_a? String
+          @writer.print content.gsub('&', '&amp;')
+          close!(false)
+        else
+          if block
+            instance_eval(&block)
+            close!
+          end
+        end
+      end
+
+      def open!(name, attrs = {})
+        attrs = attrs.map { |attr, value| %Q( #{attr}="#{value}") }.join('')
+        @writer.print "\n" + ' ' * options[:indent_by] * @opened_tags.size
+        @opened_tags << name
+        @writer.print "<#{name}#{attrs}>"
+      end
+
+      private
+      def base_url
+        schema, host = @options[:base_url].scan(/^(https?:\/\/)?(.+?)\/?$/).flatten
+        "#{schema || 'http://'}#{host}/"
+      rescue
+         ""
+      end
+    end
+  end
+end

data/lib/massive_sitemap/builder/index.rb

@@ -0,0 +1,29 @@
+require "massive_sitemap/builder/base"
+
+module MassiveSitemap
+  module Builder
+    class Index < Base
+      HEADER_NAME       = 'sitemapindex'
+      HEADER_ATTRIBUTES = {
+        :xmlns => 'http://www.sitemaps.org/schemas/sitemap/0.9'
+      }
+
+      def initialize(writer, options = {}, &block)
+        super(writer, options) do
+          writer.each do |path, last_modified|
+            add path, :last_modified => last_modified
+          end
+        end
+      end
+
+      def add_url!(location, attrs = {})
+        init!
+
+        tag! 'sitemap' do
+          tag! 'loc', location
+          tag! 'lastmod', attrs[:last_modified].utc.strftime('%Y-%m-%dT%H:%M:%S+00:00') if attrs[:last_modified]
+        end
+      end
+    end
+  end
+end

data/lib/massive_sitemap/builder/rotating.rb

@@ -0,0 +1,53 @@
+require "massive_sitemap/builder/base"
+# writer only has print and puts as interface
+
+module MassiveSitemap
+  module Builder
+    class Rotating < Base
+      NUM_URLS = 1..50_000
+
+      def initialize(writer, options = {}, &block)
+        @max_urls  = options[:max_per_sitemap] || NUM_URLS.max
+        @rotations = 0
+        @urls = 0
+
+        unless NUM_URLS.member?(@max_urls)
+          raise ArgumentError, %Q(":max_per_sitemap" must be greater than #{NUM_URLS.min} and smaller than #{NUM_URLS.max})
+        end
+
+        super
+      end
+
+      # On rotation, close current file, and reopen a new one
+      # with same file name but -<counter> appendend
+      def init!(&block)
+        unless @writer.inited?
+          @urls = 0
+          filename = filename_with_rotation(@writer.options[:filename], @rotations)
+          @rotations += 1
+          @writer.init! :filename => filename
+          header!(&block)
+        end
+      end
+
+      def add_url!(location, attrs = {})
+        if @urls >= @max_urls
+          close!
+        end
+        super
+        @urls += 1
+      end
+
+      private
+      def filename_with_rotation(filename, rotation = nil)
+        filename, _, ext = split_filename(filename)
+        rotation = (rotation.to_i > 0) ? "-#{rotation}" : nil
+        [filename, rotation, ext].join
+      end
+
+      def split_filename(filename)
+        filename.to_s.scan(/^([^.]*?)(-[0-9]+)?(\..+)?$/).flatten
+      end
+    end
+  end
+end

data/lib/massive_sitemap/builder.rb

@@ -0,0 +1,12 @@
+require "massive_sitemap/builder/base"
+
+module MassiveSitemap
+  module Builder
+
+    def new(writer, options = {}, &block)
+      Base.new(writer, options, &block)
+    end
+    module_function :new
+
+  end
+end

data/lib/massive_sitemap/ping.rb

@@ -0,0 +1,21 @@
+module MassiveSitemap
+  class Ping
+    PING = {
+     :google => 'http://www.google.comwebmasters/tools/ping?sitemap=%s';
+     :bing   => 'http://www.bing.com/webmaster/ping.aspx?siteMap=%s',
+     :ask    => 'http://submissions.ask.com/ping?sitemap=%s'
+    }
+
+    def self.ping_search_engines(sitemap_uri, engines = [])
+      require 'net/http'
+      require 'uri'
+      require 'cgi'
+
+      sitemap_uri = CGI::escape(sitemap_uri)
+
+      Array(engines).each do |engine_url|
+        Net::HTTP.get URI.parse(engine_url % sitemap_uri)
+      end
+    end
+  end
+end

data/lib/massive_sitemap/writer/base.rb

@@ -0,0 +1,60 @@
+
+module MassiveSitemap
+  module Writer
+    class Base
+      OPTS = {}
+
+      attr_reader :options
+
+      def initialize(options = {})
+        @options = self.class::OPTS.merge(options)
+        @stream  = nil
+      end
+
+      # Interface
+      def open_stream
+        @string ||= StringIO.new
+      end
+
+      def close_stream(stream)
+      end
+
+      def init?
+        true
+      end
+
+      def streams
+        []
+      end
+
+      # API
+      def init!(options = {})
+        close!
+        @options.merge!(options)
+        if init?
+          @stream = open_stream
+        end
+      end
+
+      def close!
+        if inited?
+          close_stream(@stream)
+          @stream = nil
+        end
+      end
+
+      def inited?
+        @stream
+      end
+
+      def print(string)
+        @stream.print(string) if inited?
+      end
+
+      def each(&block)
+        streams.each(&block)
+      end
+    end
+
+  end
+end

data/lib/massive_sitemap/writer/file.rb

@@ -0,0 +1,59 @@
+require 'fileutils'
+require "massive_sitemap/writer/base"
+
+# Write into File
+
+module MassiveSitemap
+  module Writer
+    class File < Base
+
+      class FileExistsException < IOError; end
+
+      OPTS = {
+        :document_full   => '.',
+        :force_overwrite => false,
+        :filename        => "sitemap.xml",
+        :index_filename  => "sitemap_index.xml",
+      }
+
+      def open_stream
+        ::File.open(tmp_filename, 'w:ASCII-8BIT')
+      end
+
+      def close_stream(stream)
+        stream.close
+        # Move from tmp_file into acutal file
+        ::File.delete(filename) if ::File.exists?(filename)
+        ::File.rename(tmp_filename, filename)
+      end
+
+      def init?
+        if !options[:force_overwrite] && ::File.exists?(filename)
+          raise FileExistsException, "Can not create file: #{filename} exits"
+        end
+        true
+      end
+
+      def streams
+        files.map do |path|
+          next if path.include?(options[:index_filename])
+          [::File.basename(path), ::File.stat(path).mtime]
+        end.compact
+      end
+
+      private
+      def filename
+        ::File.join options[:document_full], options[:filename]
+      end
+
+      def tmp_filename
+        filename + ".tmp"
+      end
+
+      def files
+        Dir[::File.join(options[:document_full], "*.xml")]
+      end
+    end
+
+  end
+end

data/lib/massive_sitemap/writer/gzip_file.rb

@@ -0,0 +1,24 @@
+require 'zlib'
+
+require "massive_sitemap/writer/file"
+# Write into GZipped File
+
+module MassiveSitemap
+  module Writer
+
+    class GzipFile < File
+      def open_stream
+        ::Zlib::GzipWriter.new(super)
+      end
+
+      private
+      def filename
+        super + ".gz"
+      end
+
+      def files
+        Dir[::File.join(options[:document_full], "*.xml.gz")]
+      end
+    end
+  end
+end

data/lib/massive_sitemap/writer/locking_file.rb

@@ -0,0 +1,31 @@
+require 'zlib'
+
+require "massive_sitemap/writer/file"
+# Create Lock before writing to file
+
+module MassiveSitemap
+  module Writer
+
+    class LockingFile < File
+      LOCK_FILE = 'generator.lock'
+
+      def open_stream
+        ::File.open(LOCK_FILE, 'w', ::File::EXCL) #lock!
+        super
+      end
+
+      def close_stream(stream)
+        super
+        FileUtils.rm(LOCK_FILE) #unlock!
+      end
+
+      def init?
+        if ::File.exists?(LOCK_FILE)
+          raise Errno::EACCES
+        end
+        super
+      end
+    end
+
+  end
+end

data/lib/massive_sitemap/writer/string.rb

@@ -0,0 +1,29 @@
+require 'stringio'
+require "massive_sitemap/writer/base"
+
+# Write into String
+# Perfect for testing porpuses
+module MassiveSitemap
+  module Writer
+
+    class String < Base
+
+      def open_stream
+        @string ||= StringIO.new
+      end
+
+      def to_s
+        @string.string rescue ""
+      end
+
+      def ==(other_string)
+        to_s == other_string
+      end
+
+      def include?(other_string)
+        to_s.include?(other_string)
+      end
+    end
+
+  end
+end

data/lib/massive_sitemap.rb

@@ -0,0 +1,52 @@
+require 'massive_sitemap/writer/file'
+require 'massive_sitemap/writer/gzip_file'
+require 'massive_sitemap/builder/rotating'
+require 'massive_sitemap/builder/index'
+
+# Page at -> <base_url>
+# http://example.de/dir/
+
+# Index at
+# http://sitemap.example.de/index-dir/
+
+# Save at -> <document_full>
+# /root/dir/ ->  <document_root>/<document_path>
+
+module MassiveSitemap
+  DEFAULTS = {
+    # global
+    :index_base_url  => nil,
+    :gzip            => false,
+    :writer          => MassiveSitemap::Writer::File,
+
+    # writer
+    :document_full   => '.',
+    :force_overwrite => false,
+    :filename        => "sitemap.xml",
+    :index_filename  => "sitemap_index.xml",
+
+    # builder
+    :base_url        => nil,
+    :indent_by       => 2,
+  }
+
+  def generate(options = {}, &block)
+    @options = DEFAULTS.merge options
+
+    unless @options[:base_url]
+      raise ArgumentError, 'you must specify ":base_url" string'
+    end
+    @options[:index_base_url] ||= @options[:base_url]
+
+    Dir.mkdir(@options[:document_full]) unless ::File.exists?(@options[:document_full])
+
+    @options[:writer] = MassiveSitemap::Writer::GzipFile if @options[:gzip]
+
+    @writer = @options[:writer].new @options
+    Builder::Rotating.generate(@writer, @options, &block)
+
+    @writer.options.merge!(:filename => @options[:index_filename], :force_overwrite => true)
+    Builder::Index.generate(@writer, @options.merge(:base_url => @options[:index_base_url]))
+  end
+  module_function :generate
+end

data/massive_sitemap.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "massive_sitemap"
+  s.version     = File.read("VERSION").to_s.strip
+  s.authors     = ["Tobias Bielohlawek"]
+  s.email       = ["tobi@soundcloud.com"]
+  s.homepage    = "http://github.com/rngtng/massive_sitemap"
+  s.summary     = %q{Build painfree sitemaps for websites with millions of pages}
+  s.description = %q{MassiveSitemap - build huge sitemaps painfree. Differential updates keeps generation time short and reduces load on DB. It's heavealy inspired by BigSitemaps and offers compatiable API}
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  %w(rake rspec).each do |gem|
+    s.add_development_dependency *gem.split(' ')
+  end
+end

data/spec/builder/base_spec.rb

@@ -0,0 +1,149 @@
+require "spec_helper"
+
+require "massive_sitemap/builder"
+require "massive_sitemap/writer/string"
+
+describe MassiveSitemap::Builder::Base do
+  let(:header) { %Q(<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd">) }
+  let(:writer) { MassiveSitemap::Writer::String.new }
+  let(:builder) { MassiveSitemap::Builder.new(writer) }
+
+  describe "#arguments" do
+    it 'fail if no writer given' do
+      expect do
+        MassiveSitemap::Builder.new
+      end.to raise_error(ArgumentError)
+    end
+  end
+
+  context "no content added" do
+    it 'empty per default' do
+      builder
+
+      writer.should == ""
+    end
+
+    it 'generate basic skeleton' do
+      builder.init!
+      writer.should == header
+    end
+
+    it 'generate basic skeleton on double init' do
+      builder.init!
+      builder.init!
+      writer.should == header
+    end
+
+    it 'generate nothing when not inited' do
+      builder.close!
+      writer.should == ""
+    end
+
+    it "same result on double close" do
+      builder.close!
+      builder.close!
+      writer.should == ""
+    end
+
+    it "same result on double close" do
+      builder.init!
+      builder.close!
+      builder.close!
+      writer.should == %Q(#{header}\n</urlset>)
+    end
+
+    it "same result on double close" do
+      builder.init! do
+        add "test"
+      end
+      writer.should == %Q(#{header}\n  <url>\n    <loc>/test</loc>\n  </url>\n</urlset>)
+    end
+  end
+
+  context "adding content" do
+    it 'seq: generate one url' do
+      builder.add 'test'
+      builder.close!
+      writer.should == %Q(#{header}\n  <url>\n    <loc>/test</loc>\n  </url>\n</urlset>)
+    end
+  end
+
+  context "as block" do
+    it 'generate basic skeleton' do
+      MassiveSitemap::Builder.new(writer)  {}
+      writer.should == ""
+    end
+
+    it 'generate one url' do
+      MassiveSitemap::Builder.new(writer) do
+        add 'test'
+      end
+      writer.should == %Q(#{header}\n  <url>\n    <loc>/test</loc>\n  </url>\n</urlset>)
+    end
+
+    it 'generate one url with init!' do
+      MassiveSitemap::Builder.new(writer) do
+        init!
+        add 'test'
+      end
+      writer.should == %Q(#{header}\n  <url>\n    <loc>/test</loc>\n  </url>\n</urlset>)
+    end
+
+    it 'generate one url with init! block' do
+      MassiveSitemap::Builder.new(writer) do
+        init! do
+          add 'test'
+        end
+      end
+      writer.should == %Q(#{header}\n  <url>\n    <loc>/test</loc>\n  </url>\n</urlset>)
+    end
+
+    it 'generate one url with close!' do
+      MassiveSitemap::Builder.new(writer) do
+        add 'test'
+        close!
+      end
+      writer.should == %Q(#{header}\n  <url>\n    <loc>/test</loc>\n  </url>\n</urlset>)
+    end
+
+    it 'generate one url, no indent' do
+      MassiveSitemap::Builder.new(writer, :indent_by => 0) do
+        add_url! 'test'
+      end
+      writer.should == %Q(#{header}\n<url>\n<loc>test</loc>\n</url>\n</urlset>)
+    end
+
+    it 'generate two url' do
+      MassiveSitemap::Builder.new(writer) do
+        add_url! 'test'
+        add_url! 'test2'
+      end
+      writer.should == %Q(#{header}\n  <url>\n    <loc>test</loc>\n  </url>\n  <url>\n    <loc>test2</loc>\n  </url>\n</urlset>)
+    end
+
+    it 'generate one url with attrs' do
+      MassiveSitemap::Builder.new(writer, :indent_by => 0) do
+        add_url! 'test', :change_frequency => 'weekly', :priority => 0.8
+      end
+      writer.should include("<loc>test</loc>\n<changefreq>weekly</changefreq>\n<priority>0.8</priority>")
+    end
+  end
+
+  describe ".base_url" do
+    URLS = %w(
+      http://test.de/
+      test.de/
+      test.de
+    )
+
+    URLS.each do |url|
+      it "transforms to valid url" do
+        MassiveSitemap::Builder.new(writer, :base_url => url).send(:base_url).should == "http://test.de/"
+      end
+    end
+
+    it "transforms to valid url with https" do
+      MassiveSitemap::Builder.new(writer, :base_url => "https://test.de/").send(:base_url).should == "https://test.de/"
+    end
+  end
+end

data/spec/builder/index_spec.rb

@@ -0,0 +1,26 @@
+require "spec_helper"
+
+require "massive_sitemap/builder/index"
+require "massive_sitemap/writer/string"
+
+describe MassiveSitemap::Builder::Index do
+  INDEX_HEADER = %Q(<?xml version="1.0" encoding="UTF-8"?>\n<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n<sitemap>)
+
+  let(:writer) { MassiveSitemap::Writer::String.new }
+
+  before do
+    writer.stub!(:streams).and_return(['test'])
+  end
+
+  it 'generates one url' do
+    MassiveSitemap::Builder::Index.new(writer, :indent_by => 0)
+
+    writer.should == %Q(#{INDEX_HEADER}\n<loc>/test</loc>\n</sitemap>\n</sitemapindex>)
+  end
+
+  it 'include base_url' do
+    MassiveSitemap::Builder::Index.new(writer, :base_url => "test.de", :indent_by => 0)
+    writer.should include("<loc>http://test.de/test</loc>")
+  end
+
+end

data/spec/builder/rotating_spec.rb

@@ -0,0 +1,133 @@
+require "spec_helper"
+
+require "massive_sitemap/builder/rotating"
+require "massive_sitemap/writer/string"
+
+describe MassiveSitemap::Builder::Rotating do
+  let(:header) { %Q(<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd">) }
+  let(:writer) { MassiveSitemap::Writer::String.new }
+  let(:builder) { MassiveSitemap::Builder::Rotating.new(writer) }
+
+  it 'raises error when max_per_sitemap > MAX_URLS' do
+    expect do
+      MassiveSitemap::Builder::Rotating.new(writer, :max_per_sitemap => MassiveSitemap::Builder::Rotating::NUM_URLS.max + 1)
+    end.to raise_error(ArgumentError)
+  end
+
+  it 'generates one url' do
+    MassiveSitemap::Builder::Rotating.new(writer) do
+      add_url! 'test'
+    end
+    writer.should == %Q(#{header}\n  <url>\n    <loc>test</loc>\n  </url>\n</urlset>)
+  end
+
+  it 'generates two url' do
+    MassiveSitemap::Builder::Rotating.new(writer, :max_per_sitemap => 1) do
+      add_url! 'test'
+      add_url! 'test2'
+    end
+    writer.should == %Q(#{header}\n  <url>\n    <loc>test</loc>\n  </url>\n</urlset>#{header}\n  <url>\n    <loc>test2</loc>\n  </url>\n</urlset>)
+  end
+
+  context "with file" do
+    let(:filename) { 'sitemap.xml' }
+    let(:filename2) { 'sitemap-1.xml' }
+    let(:writer) { MassiveSitemap::Writer::File.new }
+
+    after do
+      FileUtils.rm(filename) rescue nil
+      FileUtils.rm(filename2) rescue nil
+    end
+
+    it 'generates two url' do
+      expect do
+        expect do
+          MassiveSitemap::Builder::Rotating.new(writer, :max_per_sitemap => 1) do
+            add 'test'
+            add 'test2'
+          end
+        end.to change { File.exists?(filename) }.to(true)
+      end.to change { File.exists?(filename2) }.to(true)
+    end
+
+    it 'generates two url when file exists' do
+      File.open(filename, 'w') {}
+      expect do
+        expect do
+          MassiveSitemap::Builder::Rotating.new(writer, :max_per_sitemap => 1) do
+            begin
+              add 'test'
+            rescue MassiveSitemap::Writer::File::FileExistsException => e
+            end
+            add 'test2'
+          end
+        end.to_not change { File.exists?(filename) }.to(true)
+      end.to change { File.exists?(filename2) }.to(true)
+    end
+  end
+
+  describe "#filename_with_rotation" do
+    context "keeps filename" do
+      it "rotation is zero" do
+        builder.send(:filename_with_rotation, "sitemap.xml").should == "sitemap.xml"
+      end
+
+      it "rotation is zero" do
+        builder.send(:filename_with_rotation, "sitemap2.xml").should == "sitemap2.xml"
+      end
+
+      it "rotation is zero" do
+        builder.send(:filename_with_rotation, "sitemap.xml", nil).should == "sitemap.xml"
+      end
+
+      it "rotation is nil" do
+        builder.send(:filename_with_rotation, "sitemap.xml", 0).should == "sitemap.xml"
+      end
+    end
+
+    context "rotation is 1" do
+      it "add prefix" do
+        builder.send(:filename_with_rotation, "sitemap.xml", 1).should == "sitemap-1.xml"
+      end
+
+      it "rotation is zero" do
+        builder.send(:filename_with_rotation, "sitemap-1.xml", 1).should == "sitemap-1.xml"
+      end
+
+      it "rotation is zero" do
+        builder.send(:filename_with_rotation, "sitemap-user.xml", 1).should == "sitemap-user-1.xml"
+      end
+    end
+  end
+
+  describe "#split_filename" do
+    FILENAMES = {
+      nil                     => ["", nil, nil],
+      ".xml"                  => ["", nil, ".xml"],
+      ".xml.gz"               => ["", nil, ".xml.gz"],
+      "sitemap"               => ["sitemap", nil, nil],
+      "sitemap.xml"           => ["sitemap", nil, ".xml"],
+      "sitemap.xml.gz"        => ["sitemap", nil, ".xml.gz"],
+      "-1.xml"                => ["", "-1", ".xml"],
+      "-1.xml.gz"             => ["", "-1", ".xml.gz"],
+      "sitemap-1"             => ["sitemap", "-1", nil],
+      "sitemap-1.xml"         => ["sitemap", "-1", ".xml"],
+      "sitemap-1.xml.gz"      => ["sitemap", "-1", ".xml.gz"],
+      "-user-1.xml"           => ["-user", "-1", ".xml"],
+      "-user-1.xml.gz"        => ["-user", "-1", ".xml.gz"],
+      "sitemap-user-1"        => ["sitemap-user", "-1", nil],
+      "sitemap-user-1.xml"    => ["sitemap-user", "-1", ".xml"],
+      "sitemap-user-1.xml.gz" => ["sitemap-user", "-1", ".xml.gz"],
+      "sitemap1"              => ["sitemap1", nil, nil],
+      "sitemap1.xml"          => ["sitemap1", nil, ".xml"],
+      "sitemap1.xml.gz"       => ["sitemap1", nil, ".xml.gz"],
+    }
+
+    FILENAMES.each do |filename, expected|
+      it "splits filename #{filename} into #{expected.join(' ')}" do
+        builder.send(:split_filename, filename).should == expected
+      end
+    end
+  end
+
+end

data/spec/massive_sitemap_spec.rb

@@ -0,0 +1,158 @@
+require "spec_helper"
+
+require "massive_sitemap/writer/gzip_file"
+
+describe MassiveSitemap do
+  let(:index_filename) { 'sitemap_index.xml' }
+  let(:filename) { 'sitemap.xml' }
+  let(:filename2) { 'sitemap2.xml' }
+
+  def output(file = filename)
+    `cat '#{file}'`
+  end
+
+  def gz_filename(file = filename)
+    "#{file}.gz"
+  end
+
+  after do
+    FileUtils.rm(index_filename) rescue nil
+    FileUtils.rm(filename) rescue nil
+    FileUtils.rm(filename2) rescue nil
+  end
+
+  describe "#initalize" do
+    it 'fail if no base_url given' do
+      expect do
+        MassiveSitemap.generate
+      end.to raise_error(ArgumentError)
+    end
+
+    it "does not create empty sitemap file" do
+      expect do
+        MassiveSitemap.generate(:base_url => 'test.de/')
+      end.to_not change { ::File.exists?(filename) }
+    end
+
+    context "custom writer" do
+      after do
+        FileUtils.rm(gz_filename(index_filename)) rescue nil
+        FileUtils.rm(gz_filename) rescue nil
+      end
+
+      it 'takes gzips writer' do
+        expect do
+          MassiveSitemap.generate(:base_url => 'test.de/', :gzip => true) do
+             add "dummy"
+          end
+        end.to change { ::File.exists?(gz_filename) }.to(true)
+      end
+
+      it 'takes custom writer' do
+        expect do
+          MassiveSitemap.generate(:base_url => 'test.de/', :writer => MassiveSitemap::Writer::GzipFile) do
+            add "dummy"
+          end
+        end.to change { ::File.exists?(gz_filename) }.to(true)
+      end
+    end
+  end
+
+  describe "#generate" do
+    it 'adds url' do
+      MassiveSitemap.generate(:base_url => 'test.de') do
+        add "track/name"
+      end
+      output.should include("<loc>http://test.de/track/name</loc>")
+    end
+
+    it 'adds url with root slash' do
+      MassiveSitemap.generate(:base_url => 'test.de/') do
+        add "/track/name"
+      end
+      output.should include("<loc>http://test.de/track/name</loc>")
+    end
+
+    it "doesn't fail for existing file" do
+      File.open(filename, 'w') {}
+      expect do
+        MassiveSitemap.generate(:base_url => 'test.de/') do
+          add "/track/name"
+        end
+      end.to_not change { File.stat(filename).mtime }
+    end
+
+    context 'nested generation' do
+      it 'adds url of nested builder' do
+        MassiveSitemap.generate(:base_url => 'test.de/') do
+          writer = @writer.class.new(@options.merge(:filename => 'sitemap2.xml'))
+          MassiveSitemap::Builder::Rotating.new(writer, @options) do
+            add "/set/name"
+          end
+        end
+        output(filename2).should include("<loc>http://test.de/set/name</loc>")
+      end
+
+      it 'executes block altough first sitemap exists' do
+        File.open(filename, 'w') {}
+        MassiveSitemap.generate(:base_url => 'test.de/') do
+          writer = @writer.class.new(@options.merge(:filename => 'sitemap2.xml'))
+          MassiveSitemap::Builder::Rotating.new(writer, @options) do
+            add "/set/name"
+          end
+        end
+        output(filename2).should include("<loc>http://test.de/set/name</loc>")
+      end
+    end
+
+  end
+
+  describe "#generate_index" do
+    let(:lastmod) { File.stat(index_filename).mtime.utc.strftime('%Y-%m-%dT%H:%M:%S+00:00') }
+
+    it "does not create empty files" do
+      MassiveSitemap.generate(:base_url => 'test.de/')
+      ::File.exists?(index_filename).should be_false
+    end
+
+    it 'includes urls' do
+      MassiveSitemap.generate(:base_url => 'test.de/', :indent_by => 0) do
+        add "dummy"
+      end
+
+      output(index_filename).should include("<sitemap>\n<loc>http://test.de/sitemap.xml</loc>\n<lastmod>#{lastmod}</lastmod>\n</sitemap>")
+    end
+
+    it 'includes index base url' do
+      MassiveSitemap.generate(:base_url => 'test.de/', :index_base_url => 'index.de/')  do
+        add "dummy"
+      end
+
+      output(index_filename).should include("<loc>http://index.de/sitemap.xml</loc>")
+    end
+
+    it 'overwrites existing one' do
+      File.open(index_filename, 'w') {}
+      MassiveSitemap.generate(:base_url => 'test.de/', :index_base_url => 'index.de/') do
+        add "dummy"
+      end
+
+      output(index_filename).should include("<loc>http://index.de/sitemap.xml</loc>")
+    end
+
+    context "gziped" do
+      after do
+        FileUtils.rm(gz_filename(index_filename)) rescue nil
+        FileUtils.rm(gz_filename) rescue nil
+      end
+
+      it 'creates sitemap file' do
+        expect do
+          MassiveSitemap.generate(:base_url => 'test.de/', :writer => MassiveSitemap::Writer::GzipFile) do
+            add "dummy"
+          end
+        end.to change { ::File.exists?(gz_filename(index_filename)) }.to(true)
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+$:.unshift File.expand_path("../../lib", __FILE__)
+require "massive_sitemap"

data/spec/writer/file_spec.rb

@@ -0,0 +1,104 @@
+require "spec_helper"
+
+require "massive_sitemap/writer/file"
+
+describe MassiveSitemap::Writer::File do
+  let(:filename) { 'sitemap.xml' }
+  let(:filename2) { 'sitemap-1.xml' }
+  let(:writer) {  MassiveSitemap::Writer::File.new.tap { |w| w.init! } }
+
+  after do
+    FileUtils.rm(filename) rescue nil
+    FileUtils.rm(filename2) rescue nil
+  end
+
+  describe "document_full" do
+    let(:folder) { "test" }
+
+    before do
+      Dir.mkdir(folder) unless ::File.exists?(folder)
+    end
+
+    after do
+      FileUtils.rm_rf(folder) rescue nil
+    end
+
+    it 'appends document_full' do
+      expect do
+        MassiveSitemap::Writer::File.new(:document_full => folder).tap do |w|
+          w.init!
+          w.close!
+        end
+      end.to change { File.exists?("test/#{filename}") }.to(true)
+    end
+
+    it 'appends document_full' do
+      expect do
+        MassiveSitemap::Writer::File.new(:document_full => "#{folder}/").tap do |w|
+          w.init!
+          w.close!
+        end
+      end.to change { File.exists?("test/#{filename}") }.to(true)
+    end
+  end
+
+  it 'create file' do
+    expect do
+      writer.close!
+    end.to change { File.exists?(filename) }.to(true)
+  end
+
+  it 'create second file on rotation' do
+    expect do
+      expect do
+        writer.close!
+      end.to change { File.exists?(filename) }.to(true)
+      writer.init!(:filename => filename2)
+      writer.close!
+    end.to change { File.exists?(filename2) }.to(true)
+  end
+
+  it 'write into file' do
+    writer.print 'test'
+    writer.close!
+    `cat '#{filename}'`.should == "test"
+  end
+
+  it 'init new file closes current' do
+    writer.print 'test'
+    writer.init!(:filename => filename2)
+    `cat '#{filename}'`.should == "test"
+  end
+
+  it 'write into second file' do
+    writer.print 'test'
+    writer.init!(:filename => filename2)
+    writer.print 'test2'
+    writer.close!
+    `cat '#{filename2}'`.should == "test2"
+  end
+
+  context "opening write file" do
+    before do
+      File.open(filename, 'w') {}
+    end
+
+    after do
+      FileUtils.rm(filename) rescue nil
+    end
+
+    it 'raises when file exits' do
+      writer = MassiveSitemap::Writer::File.new
+      expect do
+        writer.init!
+      end.to raise_error(MassiveSitemap::Writer::File::FileExistsException)
+    end
+
+    it 'raises when file exits' do
+      writer = MassiveSitemap::Writer::File.new(:force_overwrite => true)
+      expect do
+        writer.init!
+      end.to_not raise_error(MassiveSitemap::Writer::File::FileExistsException)
+    end
+  end
+end

data/spec/writer/gzip_file_spec.rb

@@ -0,0 +1,20 @@
+require "massive_sitemap/writer/gzip_file"
+
+describe MassiveSitemap::Writer::GzipFile do
+  let(:filename) { 'sitemap.xml' }
+  let(:tmp_filename) { "#{filename}.tmp" }
+  let(:gz_filename) { "#{filename}.gz" }
+  let(:writer) { MassiveSitemap::Writer::GzipFile.new.tap { |w| w.init! } }
+
+  after do
+    FileUtils.rm(filename) rescue nil
+    FileUtils.rm(tmp_filename) rescue nil
+    FileUtils.rm(gz_filename) rescue nil
+  end
+
+  it 'creates gzip file' do
+    expect do
+      writer.close!
+    end.to change { File.exists?(gz_filename) }.from(false).to(true)
+  end
+end

data/spec/writer/locking_file_spec.rb

@@ -0,0 +1,34 @@
+require "massive_sitemap/writer/locking_file"
+
+describe MassiveSitemap::Writer::LockingFile do
+  let(:filename) { 'sitemap.xml' }
+  let(:tmp_filename) { "#{filename}.tmp" }
+  let(:lock_file) {  MassiveSitemap::Writer::LockingFile::LOCK_FILE }
+  let(:writer) { MassiveSitemap::Writer::LockingFile.new.tap { |w| w.init! } }
+
+  after do
+    FileUtils.rm(filename) rescue nil
+    FileUtils.rm(tmp_filename) rescue nil
+    FileUtils.rm(lock_file) rescue nil
+  end
+
+  it 'creates lockfile' do
+    expect do
+      writer
+    end.to change { File.exists?(lock_file) }.to(true)
+  end
+
+  it 'deletes lockfile' do
+    writer
+    expect do
+      writer.close!
+    end.to change { File.exists?(lock_file) }.to(false)
+  end
+
+  it 'fails if lockfile exists' do
+    File.open(lock_file, 'w') {}
+    expect do
+      writer
+    end.to raise_error
+  end
+end

metadata

@@ -0,0 +1,105 @@
+--- !ruby/object:Gem::Specification
+name: massive_sitemap
+version: !ruby/object:Gem::Version
+  version: 2.0.0.rc1
+  prerelease: 6
+platform: ruby
+authors:
+- Tobias Bielohlawek
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-02-12 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &70297299930500 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70297299930500
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70297299930040 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70297299930040
+description: MassiveSitemap - build huge sitemaps painfree. Differential updates keeps
+  generation time short and reduces load on DB. It's heavealy inspired by BigSitemaps
+  and offers compatiable API
+email:
+- tobi@soundcloud.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- CHANGELOG.md
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- VERSION
+- lib/massive_sitemap.rb
+- lib/massive_sitemap/builder.rb
+- lib/massive_sitemap/builder/base.rb
+- lib/massive_sitemap/builder/index.rb
+- lib/massive_sitemap/builder/rotating.rb
+- lib/massive_sitemap/ping.rb
+- lib/massive_sitemap/writer/base.rb
+- lib/massive_sitemap/writer/file.rb
+- lib/massive_sitemap/writer/gzip_file.rb
+- lib/massive_sitemap/writer/locking_file.rb
+- lib/massive_sitemap/writer/string.rb
+- massive_sitemap.gemspec
+- spec/builder/base_spec.rb
+- spec/builder/index_spec.rb
+- spec/builder/rotating_spec.rb
+- spec/massive_sitemap_spec.rb
+- spec/spec_helper.rb
+- spec/writer/file_spec.rb
+- spec/writer/gzip_file_spec.rb
+- spec/writer/locking_file_spec.rb
+homepage: http://github.com/rngtng/massive_sitemap
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>'
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.15
+signing_key: 
+specification_version: 3
+summary: Build painfree sitemaps for websites with millions of pages
+test_files:
+- spec/builder/base_spec.rb
+- spec/builder/index_spec.rb
+- spec/builder/rotating_spec.rb
+- spec/massive_sitemap_spec.rb
+- spec/spec_helper.rb
+- spec/writer/file_spec.rb
+- spec/writer/gzip_file_spec.rb
+- spec/writer/locking_file_spec.rb