big_sitemap 0.5.1

data/History.txt

@@ -0,0 +1,62 @@
+=== 0.5.1 / 2009-09-07
+
+* Fixes an issue with the :last_modified key being passed into the find method options
+
+=== 0.5.0 / 2009-09-07
+
+* Add support for lambdas when specifying lastmod
+
+=== 0.4.0 / 2009-08-09
+
+* Use Bing instead of Live/MSN.  Note, this breaks backwards compatibility as
+  the old :ping_msn option is now :ping_bing.
+
+=== 0.3.5 / 2009-08-05
+
+* Fixed bugs in root_url generation and url_for_sitemap generation
+
+=== 0.3.4 / 2009-07-02
+
+* BigSitemap-specific options are no longer passed through to the ORM's find method
+
+=== 0.3.2 / 2009-06-09
+
+* Better handling of URLs when Rails' polymorphic_url isn't available in the model
+
+=== 0.3.2 / 2009-06-09
+
+* Fixes "uninitialized constant ActionController" error
+* Fixes "Unknown key(s): path" error
+
+=== 0.3.1 / 2009-04-18
+
+* Fixes broken gemspec
+
+=== 0.3.0 / 2009-04-06
+
+* API change: Pass model through as first argument to add method, e.g.sitemap.add(Posts, {:path => 'articles'})
+* API change: Use Rails' polymorphic_url helper to generate URLs if Rails is being used
+* API change: Only ping search engines when ping_search_engines is explicitly called
+* Add support for passing options through to the model's find method, e.g. :conditions
+* Allow base URL to be specified as a hash as well as a string
+* Add support for changefreq and priority
+* Pluralize sitemap model filenames
+* GZipping may optionally be turned off
+
+=== 0.2.1 / 2009-03-12
+
+* Normalize path arguments so it no longer matters whether a leading slash is used or not
+
+=== 0.2.0 / 2009-03-11
+
+* Methods are now chainable
+
+=== 0.1.4 / 2009-03-11
+
+* Add clean method to clear out Sitemaps directory
+* Make methods chainable
+
+=== 0.1.3 / 2009-03-10
+
+* Initial release
+

data/LICENSE

@@ -0,0 +1,22 @@
+(The MIT License)
+
+Copyright (c) 2009 Stateless Systems (http://statelesssystems.com)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,117 @@
+= BigSitemap
+
+BigSitemap is a Sitemap (http://sitemaps.org) generator suitable for applications with greater than 50,000 URLs.  It splits large Sitemaps into multiple files, gzips the files to minimize bandwidth usage, batches database queries to minimize memory usage, can be set up with just a few lines of code and is compatible with just about any framework.
+
+BigSitemap is best run periodically through a Rake/Thor task.
+
+  require 'big_sitemap'
+
+  sitemap = BigSitemap.new(:url_options => {:host => 'example.com'})
+
+  # Add a model
+  sitemap.add Product
+
+  # Add another model with some options
+  sitemap.add(Post, {
+    :conditions       => {:published => true},
+    :path             => 'articles',
+    :change_frequency => 'daily',
+    :priority         => 0.5
+  })
+
+  # Generate the files
+  sitemap.generate
+
+The code above will create a minimum of three files:
+
+1. public/sitemaps/sitemap_index.xml.gz
+2. public/sitemaps/sitemap_products.xml.gz
+3. public/sitemaps/sitemap_posts.xml.gz
+
+If your sitemaps grow beyond 50,000 URLs (this limit can be overridden with the <code>:max_per_sitemap</code> option), the sitemap files will be partitioned into multiple files (<code>sitemap_products_1.xml.gz</code>, <code>sitemap_products_2.xml.gz</code>, ...).
+
+If you're using Rails then the URLs for each database record are generated with the <code>polymorphic_url</code> helper.  That means that the URL for a record will be exactly what you would expect: generated with respect to the routing setup of your app.  In other contexts where this helper isn't available, the URLs are generated in the form:
+
+  :base_url/:path/:to_param
+
+If the <code>to_param</code> method does not exist, then <code>id</code> will be used.
+
+== Install
+
+Via gem:
+
+  sudo gem install alexrabarts-big_sitemap -s http://gems.github.com
+
+== Advanced
+
+=== Options
+
+* <code>:url_options</code> -- hash with <code>:host</code>, optionally <code>:port</code> and <code>:protocol</code>
+* <code>:base_url</code> -- string alternative to <code>:url_options</code>, e.g. "https://example.com:8080/"
+* <code>:document_root</code> -- string defaults to <code>Rails.root</code> or <code>Merb.root</code> if available
+* <code>:path</code> -- string defaults to 'sitemaps', which places sitemap files under the <code>/sitemaps</code> directory
+* <code>:max_per_sitemap</code> -- <code>50000</code>, which is the limit dictated by Google but can be less
+* <code>:batch_size</code> -- <code>1001</code> (not <code>1000</code> due to a bug in DataMapper)
+* <code>:gzip</code> -- <code>true</code>
+* <code>:ping_google</code> -- <code>true</code>
+* <code>:ping_yahoo</code> -- <code>false</code>, needs <code>:yahoo_app_id</code>
+* <code>:ping_bing</code> -- <code>false</code>
+* <code>:ping_ask</code> -- <code>false</code>
+
+=== Chaining
+
+You can chain methods together.  You could even get away with as little code as:
+
+  BigSitemap.new(:url_options => {:host => 'example.com'}).add(Post).generate
+
+=== Pinging Search Engines
+
+To ping search engines, call <code>ping_search_engines</code> after you generate the sitemap:
+
+  sitemap.generate
+  sitemap.ping_search_engines
+
+=== Change Frequency, Priority and Last Modified
+
+You can control "changefreq", "priority" and "lastmod" values for each record individually by passing lambdas instead of fixed values:
+
+  sitemap.add(Posts,
+    :change_frequency => lambda {|post| ... },
+    :priority         => lambda {|post| ... },
+    :last_modified    => lambda {|post| ... }
+  )
+
+=== Find Methods
+
+Your models must provide either a <code>find_for_sitemap</code> or <code>all</code> class method that returns the instances that are to be included in the sitemap.
+
+Additionally, you models must provide a <code>count_for_sitemap</code> or <code>count</code> class method that returns a count of the instances to be included.
+
+If you're using ActiveRecord (Rails) or DataMapper then <code>all</code> and <code>count</code> are already provided and you don't need to do anything unless you want to include a subset of records.  If you provide your own <code>find_for_sitemap</code> or <code>all</code> method then it should be able to handle the <code>:offset</code> and <code>:limit</code> options, in the same way that ActiveRecord and DataMapper handle them.  This is especially important if you have more than 50,000 URLs.
+
+=== Cleaning the Sitemaps Directory
+
+Calling the <code>clean</code> method will remove all files from the Sitemaps directory.
+
+== Limitations
+
+If your database is likely to shrink during the time it takes to create the sitemap then you might run into problems (the final, batched SQL select will overrun by setting a limit that is too large since it is calculated from the count, which is queried at the very beginning).  Patches welcome!
+
+== TODO
+
+Tests for Rails components.
+
+== Credits
+
+Thanks to Alastair Brunton and Harry Love, who's work provided a starting point for this library.
+http://scoop.cheerfactory.co.uk/2008/02/26/google-sitemap-generator/
+
+Thanks to those who have contributed patches:
+
+* Mislav Marohnić
+* Jeff Schoolcraft
+* Dalibor Nasevic
+
+== Copyright
+
+Copyright (c) 2009 Stateless Systems (http://statelesssystems.com). See LICENSE for details.

data/VERSION.yml

@@ -0,0 +1,4 @@
+--- 
+:patch: 1
+:major: 0
+:minor: 5

data/lib/big_sitemap.rb

@@ -0,0 +1,239 @@
+require 'uri'
+require 'big_sitemap/builder'
+require 'extlib'
+require 'action_controller' if defined? Rails
+
+class BigSitemap
+  DEFAULTS = {
+    :max_per_sitemap => Builder::MAX_URLS,
+    :batch_size      => 1001,
+    :path            => 'sitemaps',
+    :gzip            => true,
+
+    # opinionated
+    :ping_google => true,
+    :ping_yahoo  => false, # needs :yahoo_app_id
+    :ping_bing   => false,
+    :ping_ask    => false
+  }
+
+  COUNT_METHODS     = [:count_for_sitemap, :count]
+  FIND_METHODS      = [:find_for_sitemap, :all]
+  TIMESTAMP_METHODS = [:updated_at, :updated_on, :updated, :created_at, :created_on, :created]
+  PARAM_METHODS     = [:to_param, :id]
+
+  include ActionController::UrlWriter if defined? Rails
+
+  def initialize(options)
+    @options = DEFAULTS.merge options
+
+    # Use Rails' default_url_options if available
+    @default_url_options = defined?(Rails) ? default_url_options : {}
+
+    if @options[:max_per_sitemap] <= 1
+      raise ArgumentError, '":max_per_sitemap" must be greater than 1'
+    end
+
+    if @options[:url_options]
+      @default_url_options.update @options[:url_options]
+    elsif @options[:base_url]
+      uri = URI.parse(@options[:base_url])
+      @default_url_options[:host]     = uri.host
+      @default_url_options[:port]     = uri.port
+      @default_url_options[:protocol] = uri.scheme
+    else
+      raise ArgumentError, 'you must specify either ":url_options" hash or ":base_url" string'
+    end
+
+    if @options[:batch_size] > @options[:max_per_sitemap]
+      raise ArgumentError, '":batch_size" must be less than ":max_per_sitemap"'
+    end
+
+    @options[:document_root] ||= begin
+      if defined? Rails
+        "#{Rails.root}/public"
+      elsif defined? Merb
+        "#{Merb.root}/public"
+      end
+    end
+
+    unless @options[:document_root]
+      raise ArgumentError, 'Document root must be specified with the ":document_root" option'
+    end
+
+    @file_path = "#{@options[:document_root]}/#{strip_leading_slash(@options[:path])}"
+    Dir.mkdir(@file_path) unless File.exists? @file_path
+
+    @sources       = []
+    @sitemap_files = []
+  end
+
+  def add(model, options={})
+    options[:path] ||= Extlib::Inflection.tableize(model.to_s)
+    @sources << [model, options.dup]
+    return self
+  end
+
+  def clean
+    Dir["#{@file_path}/sitemap_*.{xml,xml.gz}"].each do |file|
+      FileUtils.rm file
+    end
+    return self
+  end
+
+  def generate
+    for model, options in @sources
+      with_sitemap(Extlib::Inflection::tableize(model.to_s)) do |sitemap|
+        count_method = pick_method(model, COUNT_METHODS)
+        find_method  = pick_method(model, FIND_METHODS)
+        raise ArgumentError, "#{model} must provide a count_for_sitemap class method" if count_method.nil?
+        raise ArgumentError, "#{model} must provide a find_for_sitemap class method" if find_method.nil?
+
+        count        = model.send(count_method)
+        num_sitemaps = 1
+        num_batches  = 1
+
+        if count > @options[:batch_size]
+          num_batches  = (count.to_f / @options[:batch_size].to_f).ceil
+          num_sitemaps = (count.to_f / @options[:max_per_sitemap].to_f).ceil
+        end
+        batches_per_sitemap = num_batches.to_f / num_sitemaps.to_f
+
+        find_options = options.except(:path, :num_items, :priority, :change_frequency, :last_modified)
+
+        for sitemap_num in 1..num_sitemaps
+          # Work out the start and end batch numbers for this sitemap
+          batch_num_start = sitemap_num == 1 ? 1 : ((sitemap_num * batches_per_sitemap).ceil - batches_per_sitemap + 1).to_i
+          batch_num_end   = (batch_num_start + [batches_per_sitemap, num_batches].min).floor - 1
+
+          for batch_num in batch_num_start..batch_num_end
+            offset       = ((batch_num - 1) * @options[:batch_size])
+            limit        = (count - offset) < @options[:batch_size] ? (count - offset - 1) : @options[:batch_size]
+            find_options.update(:limit => limit, :offset => offset) if num_batches > 1
+
+            model.send(find_method, find_options).each do |record|
+              last_mod = options[:last_modified]
+              if last_mod.is_a?(Proc)
+                last_mod = last_mod.call(record)
+              elsif last_mod.nil?
+                last_mod_method = pick_method(record, TIMESTAMP_METHODS)
+                last_mod = last_mod_method.nil? ? Time.now : record.send(last_mod_method)
+              end
+
+              param_method = pick_method(record, PARAM_METHODS)
+
+              location = defined?(Rails) ? polymorphic_url(record) : nil rescue nil
+              location ||= "#{root_url}/#{strip_leading_slash(options[:path])}/#{record.send(param_method)}"
+
+              change_frequency = options[:change_frequency] || 'weekly'
+              freq = change_frequency.is_a?(Proc) ? change_frequency.call(record) : change_frequency
+
+              priority = options[:priority]
+              pri = priority.is_a?(Proc) ? priority.call(record) : priority
+
+              sitemap.add_url!(location, last_mod, freq, pri)
+            end
+          end
+        end
+      end
+    end
+
+    generate_sitemap_index
+
+    return self
+  end
+
+  def ping_search_engines
+    require 'net/http'
+    require 'cgi'
+
+    sitemap_uri = CGI::escape(url_for_sitemap(@sitemap_files.last))
+
+    if @options[:ping_google]
+      Net::HTTP.get('www.google.com', "/webmasters/tools/ping?sitemap=#{sitemap_uri}")
+    end
+
+    if @options[:ping_yahoo]
+      if @options[:yahoo_app_id]
+        Net::HTTP.get(
+          'search.yahooapis.com', "/SiteExplorerService/V1/updateNotification?" +
+            "appid=#{@options[:yahoo_app_id]}&url=#{sitemap_uri}"
+        )
+      else
+        $stderr.puts 'unable to ping Yahoo: no ":yahoo_app_id" provided'
+      end
+    end
+
+    if @options[:ping_bing]
+      Net::HTTP.get('www.bing.com', "/webmaster/ping.aspx?siteMap=#{sitemap_uri}")
+    end
+
+    if @options[:ping_ask]
+      Net::HTTP.get('submissions.ask.com', "/ping?sitemap=#{sitemap_uri}")
+    end
+  end
+
+  def root_url
+    @root_url ||= begin
+      url = ''
+      url << (@default_url_options[:protocol] || 'http')
+      url << '://' unless url.match('://')
+      url << @default_url_options[:host]
+      url << ":#{port}" if port = @default_url_options[:port] and port != 80
+      url
+    end
+  end
+
+  private
+
+  def with_sitemap(name, options={})
+    options[:index] = name == 'index'
+    options[:filename] = "#{@file_path}/sitemap_#{name}"
+    options[:max_urls] = @options[:max_per_sitemap]
+
+    unless options[:gzip] = @options[:gzip]
+      options[:indent] = 2
+    end
+
+    sitemap = Builder.new(options)
+
+    begin
+      yield sitemap
+    ensure
+      sitemap.close!
+      @sitemap_files.concat sitemap.paths!
+    end
+  end
+
+  def strip_leading_slash(str)
+    str.sub(/^\//, '')
+  end
+
+  def pick_method(model, candidates)
+    method = nil
+    candidates.each do |candidate|
+      if model.respond_to? candidate
+        method = candidate
+        break
+      end
+    end
+    method
+  end
+
+  def url_for_sitemap(path)
+    if @options[:path].blank?
+      "#{root_url}/#{File.basename(path)}"
+    else
+      "#{root_url}/#{@options[:path]}/#{File.basename(path)}"
+    end
+  end
+
+  # Create a sitemap index document
+  def generate_sitemap_index
+    with_sitemap 'index' do |sitemap|
+      for path in @sitemap_files
+        sitemap.add_url!(url_for_sitemap(path), File.stat(path).mtime)
+      end
+    end
+  end
+end

data/lib/big_sitemap/builder.rb

@@ -0,0 +1,124 @@
+require 'builder'
+require 'zlib'
+
+class BigSitemap
+  class Builder < Builder::XmlMarkup
+    NAMESPACE = 'http://www.sitemaps.org/schemas/sitemap/0.9'
+    MAX_URLS = 50000
+
+    def initialize(options)
+      @gzip = options.delete(:gzip)
+      @max_urls = options.delete(:max_urls) || MAX_URLS
+      @index = options.delete(:index)
+      @paths = []
+      @parts = 0
+
+      if @filename = options.delete(:filename)
+        options[:target] = _get_writer
+      end
+
+      super(options)
+
+      @opened_tags = []
+      _init_document
+    end
+
+    def add_url!(url, time = nil, frequency = nil, priority = nil)
+      _rotate if @max_urls == @urls
+
+      tag!(@index ? 'sitemap' : 'url') do
+        loc url
+        # W3C format is the subset of ISO 8601
+        lastmod(time.utc.strftime('%Y-%m-%dT%H:%M:%S+00:00')) unless time.nil?
+        changefreq(frequency) unless frequency.nil?
+        priority(priority) unless priority.nil?
+      end
+      @urls += 1
+    end
+
+    def close!
+      _close_document
+      target!.close if target!.respond_to?(:close)
+    end
+
+    def paths!
+      @paths
+    end
+
+    private
+
+    def _get_writer
+      if @filename
+        filename = @filename.dup
+        filename << "_#{@parts}" if @parts > 0
+        filename << '.xml'
+        filename << '.gz' if @gzip
+        _open_writer(filename)
+      else
+        target!
+      end
+    end
+
+    def _open_writer(filename)
+      file = File.open(filename, 'w+')
+      @paths << filename
+      @gzip ? Zlib::GzipWriter.new(file) : file
+    end
+
+    def _init_document
+      @urls = 0
+      instruct!
+      _open_tag(@index ? 'sitemapindex' : 'urlset', :xmlns => NAMESPACE)
+    end
+
+    def _rotate
+      # write out the current document and start writing into a new file
+      close!
+      @parts += 1
+      @target = _get_writer
+      _init_document
+    end
+
+    # add support for:
+    #   xml.open_foo!(attrs)
+    #   xml.close_foo!
+    def method_missing(method, *args, &block)
+      if method.to_s =~ /^(open|close)_(.+)!$/
+        operation, name = $1, $2
+        name = "#{name}:#{args.shift}" if Symbol === args.first
+
+        if 'open' == operation
+          _open_tag(name, args.first)
+        else
+          _close_tag(name)
+        end
+      else
+        super
+      end
+    end
+
+    # opens a tag, bumps up level but doesn't require a block
+    def _open_tag(name, attrs)
+      _indent
+      _start_tag(name, attrs)
+      _newline
+      @level += 1
+      @opened_tags << name
+    end
+
+    # closes a tag block by decreasing the level and inserting a close tag
+    def _close_tag(name)
+      @opened_tags.pop
+      @level -= 1
+      _indent
+      _end_tag(name)
+      _newline
+    end
+
+    def _close_document
+      for name in @opened_tags.reverse
+        _close_tag(name)
+      end
+    end
+  end
+end

data/test/big_sitemap_test.rb

@@ -0,0 +1,294 @@
+require File.dirname(__FILE__) + '/test_helper'
+require 'nokogiri'
+
+class BigSitemapTest < Test::Unit::TestCase
+  def setup
+    delete_tmp_files
+  end
+
+  def teardown
+    delete_tmp_files
+  end
+
+  should 'raise an error if the :base_url option is not specified' do
+    assert_nothing_raised { BigSitemap.new(:base_url => 'http://example.com', :document_root => tmp_dir) }
+    assert_raise(ArgumentError) { BigSitemap.new(:document_root => tmp_dir) }
+  end
+
+  should 'generate the same base URL' do
+    options = {:document_root => tmp_dir}
+    assert_equal(
+      BigSitemap.new(options.merge(:base_url => 'http://example.com')).root_url,
+      BigSitemap.new(options.merge(:url_options => {:host => 'example.com'})).root_url
+    )
+  end
+
+  should 'generate a sitemap index file' do
+    generate_sitemap_files
+    assert File.exists?(sitemaps_index_file)
+  end
+
+  should 'generate a single sitemap model file' do
+    create_sitemap
+    add_model
+    @sitemap.generate
+    assert File.exists?(first_sitemaps_model_file), "#{first_sitemaps_model_file} exists"
+  end
+
+  should 'generate two sitemap model files' do
+    generate_two_model_sitemap_files
+    assert File.exists?(first_sitemaps_model_file), "#{first_sitemaps_model_file} exists"
+    assert File.exists?(second_sitemaps_model_file), "#{second_sitemaps_model_file} exists"
+    assert !File.exists?(third_sitemaps_model_file), "#{third_sitemaps_model_file} does not exist"
+  end
+
+  context 'Sitemap index file' do
+    should 'contain one sitemapindex element' do
+      generate_sitemap_files
+      assert_equal 1, num_elements(sitemaps_index_file, 'sitemapindex')
+    end
+
+    should 'contain one sitemap element' do
+      generate_sitemap_files
+      assert_equal 1, num_elements(sitemaps_index_file, 'sitemap')
+    end
+
+    should 'contain one loc element' do
+      generate_one_sitemap_model_file
+      assert_equal 1, num_elements(sitemaps_index_file, 'loc')
+    end
+
+    should 'contain one lastmod element' do
+      generate_one_sitemap_model_file
+      assert_equal 1, num_elements(sitemaps_index_file, 'lastmod')
+    end
+
+    should 'contain two loc elements' do
+      generate_two_model_sitemap_files
+      assert_equal 2, num_elements(sitemaps_index_file, 'loc')
+    end
+
+    should 'contain two lastmod elements' do
+      generate_two_model_sitemap_files
+      assert_equal 2, num_elements(sitemaps_index_file, 'lastmod')
+    end
+
+    should 'not be gzipped' do
+      generate_sitemap_files(:gzip => false)
+      assert File.exists?(unzipped_sitemaps_index_file)
+    end
+  end
+
+  context 'Sitemap model file' do
+    should 'contain one urlset element' do
+      generate_one_sitemap_model_file
+      assert_equal 1, num_elements(first_sitemaps_model_file, 'urlset')
+    end
+
+    should 'contain several loc elements' do
+      generate_one_sitemap_model_file
+      assert_equal default_num_items, num_elements(first_sitemaps_model_file, 'loc')
+    end
+
+    should 'contain several lastmod elements' do
+      generate_one_sitemap_model_file
+      assert_equal default_num_items, num_elements(first_sitemaps_model_file, 'lastmod')
+    end
+
+    should 'contain several changefreq elements' do
+      generate_one_sitemap_model_file
+      assert_equal default_num_items, num_elements(first_sitemaps_model_file, 'changefreq')
+    end
+
+    should 'contain several priority elements' do
+      generate_one_sitemap_model_file(:priority => 0.2)
+      assert_equal default_num_items, num_elements(first_sitemaps_model_file, 'priority')
+    end
+
+    should 'have a change frequency of weekly by default' do
+      generate_one_sitemap_model_file
+      assert_equal 'weekly', elements(first_sitemaps_model_file, 'changefreq').first.text
+    end
+
+    should 'have a change frequency of daily' do
+      generate_one_sitemap_model_file(:change_frequency => 'daily')
+      assert_equal 'daily', elements(first_sitemaps_model_file, 'changefreq').first.text
+    end
+
+    should 'be able to use a lambda to specify change frequency' do
+      generate_one_sitemap_model_file(:change_frequency => lambda {|m| m.change_frequency})
+      assert_equal TestModel.new.change_frequency, elements(first_sitemaps_model_file, 'changefreq').first.text
+    end
+
+    should 'have a priority of 0.2' do
+      generate_one_sitemap_model_file(:priority => 0.2)
+      assert_equal '0.2', elements(first_sitemaps_model_file, 'priority').first.text
+    end
+
+    should 'be able to use a lambda to specify priority' do
+      generate_one_sitemap_model_file(:priority => lambda {|m| m.priority})
+      assert_equal TestModel.new.priority.to_s, elements(first_sitemaps_model_file, 'priority').first.text
+    end
+
+    should 'be able to use a lambda to specify lastmod' do
+      generate_one_sitemap_model_file(:last_modified => lambda {|m| m.updated_at})
+      assert_equal TestModel.new.updated_at.utc.strftime('%Y-%m-%dT%H:%M:%S+00:00'), elements(first_sitemaps_model_file, 'lastmod').first.text
+    end
+
+    should 'contain two loc element' do
+      generate_two_model_sitemap_files
+      assert_equal 2, num_elements(first_sitemaps_model_file, 'loc')
+      assert_equal 2, num_elements(second_sitemaps_model_file, 'loc')
+    end
+
+    should 'contain two lastmod element' do
+      generate_two_model_sitemap_files
+      assert_equal 2, num_elements(first_sitemaps_model_file, 'lastmod')
+      assert_equal 2, num_elements(second_sitemaps_model_file, 'lastmod')
+    end
+
+    should 'contain two changefreq elements' do
+      generate_two_model_sitemap_files
+      assert_equal 2, num_elements(first_sitemaps_model_file, 'changefreq')
+      assert_equal 2, num_elements(second_sitemaps_model_file, 'changefreq')
+    end
+
+    should 'contain two priority element' do
+      generate_two_model_sitemap_files(:priority => 0.2)
+      assert_equal 2, num_elements(first_sitemaps_model_file, 'priority')
+      assert_equal 2, num_elements(second_sitemaps_model_file, 'priority')
+    end
+
+    should 'strip leading slashes from controller paths' do
+      create_sitemap
+      @sitemap.add(TestModel, :path => '/test_controller').generate
+      assert(
+        !elements(first_sitemaps_model_file, 'loc').first.text.match(/\/\/test_controller\//),
+        'URL does not contain a double-slash before the controller path'
+      )
+    end
+
+    should 'not be gzipped' do
+      generate_one_sitemap_model_file(:gzip => false)
+      assert File.exists?(unzipped_first_sitemaps_model_file)
+    end
+  end
+
+  context 'add method' do
+    should 'be chainable' do
+      create_sitemap
+      assert_equal BigSitemap, @sitemap.add(TestModel).class
+    end
+  end
+
+  context 'clean method' do
+    should 'be chainable' do
+      create_sitemap
+      assert_equal BigSitemap, @sitemap.clean.class
+    end
+
+    should 'clean all sitemap files' do
+      generate_sitemap_files
+      assert Dir.entries(sitemaps_dir).size > 2, "#{sitemaps_dir} is not empty" # ['.', '..'].size == 2
+      @sitemap.clean
+      assert_equal 2, Dir.entries(sitemaps_dir).size, "#{sitemaps_dir} is empty"
+    end
+  end
+
+  context 'generate method' do
+    should 'be chainable' do
+      create_sitemap
+      assert_equal BigSitemap, @sitemap.generate.class
+    end
+  end
+
+  private
+    def delete_tmp_files
+      FileUtils.rm_rf(sitemaps_dir)
+    end
+
+    def create_sitemap(options={})
+      @sitemap = BigSitemap.new({
+        :base_url      => 'http://example.com',
+        :document_root => tmp_dir,
+        :update_google => false
+      }.update(options))
+    end
+
+    def generate_sitemap_files(options={})
+      create_sitemap(options)
+      add_model
+      @sitemap.generate
+    end
+
+    def generate_one_sitemap_model_file(options={})
+      change_frequency = options.delete(:change_frequency)
+      priority         = options.delete(:priority)
+      create_sitemap(options.merge(:max_per_sitemap => default_num_items, :batch_size => default_num_items))
+      add_model(:change_frequency => change_frequency, :priority => priority)
+      @sitemap.generate
+    end
+
+    def generate_two_model_sitemap_files(options={})
+      change_frequency = options.delete(:change_frequency)
+      priority         = options.delete(:priority)
+      create_sitemap(options.merge(:max_per_sitemap => 2, :batch_size => 1))
+      add_model(:num_items => 4, :change_frequency => change_frequency, :priority => priority)
+      @sitemap.generate
+    end
+
+    def add_model(options={})
+      num_items = options.delete(:num_items) || default_num_items
+      TestModel.stubs(:num_items).returns(num_items)
+      @sitemap.add(TestModel, options)
+    end
+
+    def default_num_items
+      10
+    end
+
+    def sitemaps_index_file
+      "#{unzipped_sitemaps_index_file}.gz"
+    end
+
+    def unzipped_sitemaps_index_file
+      "#{sitemaps_dir}/sitemap_index.xml"
+    end
+
+    def unzipped_first_sitemaps_model_file
+      "#{sitemaps_dir}/sitemap_test_models.xml"
+    end
+
+    def first_sitemaps_model_file
+      "#{sitemaps_dir}/sitemap_test_models.xml.gz"
+    end
+
+    def second_sitemaps_model_file
+      "#{sitemaps_dir}/sitemap_test_models_1.xml.gz"
+    end
+
+    def third_sitemaps_model_file
+      "#{sitemaps_dir}/sitemap_test_model_2.xml.gz"
+    end
+
+    def sitemaps_dir
+      "#{tmp_dir}/sitemaps"
+    end
+
+    def tmp_dir
+      '/tmp'
+    end
+
+    def ns
+      {'s' => 'http://www.sitemaps.org/schemas/sitemap/0.9'}
+    end
+
+    def elements(filename, el)
+      data = Nokogiri::XML.parse(Zlib::GzipReader.open(filename).read)
+      data.search("//s:#{el}", ns)
+    end
+
+    def num_elements(filename, el)
+      elements(filename, el).size
+    end
+end

data/test/fixtures/test_model.rb

@@ -0,0 +1,34 @@
+class TestModel
+  def to_param
+    object_id
+  end
+
+  def change_frequency
+    'monthly'
+  end
+
+  def priority
+    0.8
+  end
+
+  def updated_at
+    Time.at(1000000000)
+  end
+
+  class << self
+    def count_for_sitemap
+      self.find_for_sitemap.size
+    end
+
+    def num_items
+      10
+    end
+
+    def find_for_sitemap(options={})
+      instances = []
+      num_times = options.delete(:limit) || self.num_items
+      num_times.times { instances.push(self.new) }
+      instances
+    end
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,11 @@
+require 'rubygems'
+require 'test/unit'
+require 'shoulda'
+require 'mocha'
+require 'test/fixtures/test_model'
+
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'big_sitemap'
+
+class Test::Unit::TestCase
+end

metadata

@@ -0,0 +1,108 @@
+--- !ruby/object:Gem::Specification 
+name: big_sitemap
+version: !ruby/object:Gem::Version 
+  hash: 9
+  prerelease: false
+  segments: 
+  - 0
+  - 5
+  - 1
+  version: 0.5.1
+platform: ruby
+authors: 
+- Alex Rabarts
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-09-07 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: builder
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 15
+        segments: 
+        - 2
+        - 1
+        - 2
+        version: 2.1.2
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: extlib
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 41
+        segments: 
+        - 0
+        - 9
+        - 9
+        version: 0.9.9
+  type: :runtime
+  version_requirements: *id002
+description: A Sitemap generator specifically designed for large sites (although it works equally well with small sites)
+email: alexrabarts@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+- LICENSE
+files: 
+- History.txt
+- README.rdoc
+- VERSION.yml
+- lib/big_sitemap/builder.rb
+- lib/big_sitemap.rb
+- test/big_sitemap_test.rb
+- test/fixtures/test_model.rb
+- test/test_helper.rb
+- LICENSE
+has_rdoc: true
+homepage: http://github.com/alexrabarts/big_sitemap
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --inline-source
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 2
+summary: A Sitemap generator specifically designed for large sites (although it works equally well with small sites)
+test_files: []
+