alexrabarts-big_sitemap 0.3.0 → 0.3.1

data/README.rdoc

@@ -26,7 +26,7 @@ The code above will create a minimum of three files:
 2. public/sitemaps/sitemap_products.xml.gz
 3. public/sitemaps/sitemap_posts.xml.gz
 
-If your sitemaps grow beyond 50,000 URLs (this limit can be overridden with the <code>:max_per_sitemap</code> option), the sitemap files will be partitioned into multiple files (<code>sitemap_products_1.xml.gz</code>, <code>sitemap_products_2.xml.gz</code>, &#133;).
+If your sitemaps grow beyond 50,000 URLs (this limit can be overridden with the <code>:max_per_sitemap</code> option), the sitemap files will be partitioned into multiple files (<code>sitemap_products_1.xml.gz</code>, <code>sitemap_products_2.xml.gz</code>, ...).
 
 If you're using Rails then the URLs for each database record are generated with the <code>polymorphic_url</code> helper.  That means that the URL for a record will be exactly what you would expect: generated with respect to the routing setup of your app.  In other contexts where this helper isn't available, the URLs are generated in the form:
 
@@ -44,17 +44,17 @@ Via gem:
 
 === Options
 
-  * <code>:url_options</code> -- hash with <code>:host</code>, optionally <code>:port</code> and <code>:protocol</code>
-  * <code>:base_url</code> -- string alternative to <code>:url_options</code>, e.g. "https://example.com:8080/"
-  * <code>:document_root</code> -- string defaults to <code>Rails.root</code> or <code>Merb.root</code> if available
-  * <code>:path</code> -- string defaults to 'sitemaps', which places sitemap files under the <code>/sitemaps</code> directory
-  * <code>:max_per_sitemap</code> -- <code>50000</code>, which is the limit dictated by Google but can be less
-  * <code>:batch_size</code> -- <code>1001</code> (not <code>1000</code> due to a bug in DataMapper)
-  * <code>:gzip</code> -- <code>true</code>
-  * <code>:ping_google</code> -- <code>true</code>
-  * <code>:ping_yahoo</code> -- <code>false</code>, needs <code>:yahoo_app_id</code>
-  * <code>:ping_msn</code> -- <code>false</code>
-  * <code>:pink_ask</code> -- <code>false</code>
+* <code>:url_options</code> -- hash with <code>:host</code>, optionally <code>:port</code> and <code>:protocol</code>
+* <code>:base_url</code> -- string alternative to <code>:url_options</code>, e.g. "https://example.com:8080/"
+* <code>:document_root</code> -- string defaults to <code>Rails.root</code> or <code>Merb.root</code> if available
+* <code>:path</code> -- string defaults to 'sitemaps', which places sitemap files under the <code>/sitemaps</code> directory
+* <code>:max_per_sitemap</code> -- <code>50000</code>, which is the limit dictated by Google but can be less
+* <code>:batch_size</code> -- <code>1001</code> (not <code>1000</code> due to a bug in DataMapper)
+* <code>:gzip</code> -- <code>true</code>
+* <code>:ping_google</code> -- <code>true</code>
+* <code>:ping_yahoo</code> -- <code>false</code>, needs <code>:yahoo_app_id</code>
+* <code>:ping_msn</code> -- <code>false</code>
+* <code>:pink_ask</code> -- <code>false</code>
 
 === Chaining
 

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
-:patch: 0
+:patch: 1
 :major: 0
 :minor: 3

data/lib/big_sitemap/builder.rb

@@ -0,0 +1,124 @@
+require 'builder'
+require 'zlib'
+
+class BigSitemap
+  class Builder < Builder::XmlMarkup
+    NAMESPACE = 'http://www.sitemaps.org/schemas/sitemap/0.9'
+    MAX_URLS = 50000
+
+    def initialize(options)
+      @gzip = options.delete(:gzip)
+      @max_urls = options.delete(:max_urls) || MAX_URLS
+      @index = options.delete(:index)
+      @paths = []
+      @parts = 0
+
+      if @filename = options.delete(:filename)
+        options[:target] = _get_writer
+      end
+
+      super(options)
+
+      @opened_tags = []
+      _init_document
+    end
+
+    def add_url!(url, time = nil, frequency = nil, priority = nil)
+      _rotate if @max_urls == @urls
+
+      tag!(@index ? 'sitemap' : 'url') do
+        loc url
+        # W3C format is the subset of ISO 8601
+        lastmod(time.utc.strftime('%Y-%m-%dT%H:%M:%S+00:00')) unless time.nil?
+        changefreq(frequency) unless frequency.nil?
+        priority(priority) unless priority.nil?
+      end
+      @urls += 1
+    end
+
+    def close!
+      _close_document
+      target!.close if target!.respond_to?(:close)
+    end
+
+    def paths!
+      @paths
+    end
+
+    private
+
+    def _get_writer
+      if @filename
+        filename = @filename.dup
+        filename << "_#{@parts}" if @parts > 0
+        filename << '.xml'
+        filename << '.gz' if @gzip
+        _open_writer(filename)
+      else
+        target!
+      end
+    end
+
+    def _open_writer(filename)
+      file = File.open(filename, 'w+')
+      @paths << filename
+      @gzip ? Zlib::GzipWriter.new(file) : file
+    end
+
+    def _init_document
+      @urls = 0
+      instruct!
+      _open_tag(@index ? 'sitemapindex' : 'urlset', :xmlns => NAMESPACE)
+    end
+
+    def _rotate
+      # write out the current document and start writing into a new file
+      close!
+      @parts += 1
+      @target = _get_writer
+      _init_document
+    end
+
+    # add support for:
+    #   xml.open_foo!(attrs)
+    #   xml.close_foo!
+    def method_missing(method, *args, &block)
+      if method.to_s =~ /^(open|close)_(.+)!$/
+        operation, name = $1, $2
+        name = "#{name}:#{args.shift}" if Symbol === args.first
+
+        if 'open' == operation
+          _open_tag(name, args.first)
+        else
+          _close_tag(name)
+        end
+      else
+        super
+      end
+    end
+
+    # opens a tag, bumps up level but doesn't require a block
+    def _open_tag(name, attrs)
+      _indent
+      _start_tag(name, attrs)
+      _newline
+      @level += 1
+      @opened_tags << name
+    end
+
+    # closes a tag block by decreasing the level and inserting a close tag
+    def _close_tag(name)
+      @opened_tags.pop
+      @level -= 1
+      _indent
+      _end_tag(name)
+      _newline
+    end
+
+    def _close_document
+      for name in @opened_tags.reverse
+        _close_tag(name)
+      end
+    end
+  end
+end

data/lib/big_sitemap.rb

@@ -1,11 +1,10 @@
 require 'uri'
-require 'zlib'
-require 'builder'
+require 'big_sitemap/builder'
 require 'extlib'
 
 class BigSitemap
   DEFAULTS = {
-    :max_per_sitemap => 50000,
+    :max_per_sitemap => Builder::MAX_URLS,
     :batch_size      => 1001,
     :path            => 'sitemaps',
     :gzip            => true,
@@ -30,6 +29,10 @@ class BigSitemap
     # Use Rails' default_url_options if available
     @default_url_options = defined?(Rails) ? default_url_options : {}
 
+    if @options[:max_per_sitemap] <= 1
+      raise ArgumentError, '":max_per_sitemap" must be greater than 1'
+    end
+
     if @options[:url_options]
       @default_url_options.update @options[:url_options]
     elsif @options[:base_url]
@@ -79,71 +82,55 @@ class BigSitemap
 
   def generate
     for model, options in @sources
-      count_method = pick_method(model, COUNT_METHODS)
-      find_method  = pick_method(model, FIND_METHODS)
-      raise ArgumentError, "#{model} must provide a count_for_sitemap class method" if count_method.nil?
-      raise ArgumentError, "#{model} must provide a find_for_sitemap class method" if find_method.nil?
-
-      count        = model.send(count_method)
-      num_sitemaps = 1
-      num_batches  = 1
-
-      if count > @options[:batch_size]
-        num_batches  = (count.to_f / @options[:batch_size].to_f).ceil
-        num_sitemaps = (count.to_f / @options[:max_per_sitemap].to_f).ceil
-      end
-      batches_per_sitemap = num_batches.to_f / num_sitemaps.to_f
-
-      find_options = options.dup
-
-      for sitemap_num in 1..num_sitemaps
-        # Work out the start and end batch numbers for this sitemap
-        batch_num_start = sitemap_num == 1 ? 1 : ((sitemap_num * batches_per_sitemap).ceil - batches_per_sitemap + 1).to_i
-        batch_num_end   = (batch_num_start + [batches_per_sitemap, num_batches].min).floor - 1
+      with_sitemap(Extlib::Inflection::tableize(model.to_s)) do |sitemap|
+        count_method = pick_method(model, COUNT_METHODS)
+        find_method  = pick_method(model, FIND_METHODS)
+        raise ArgumentError, "#{model} must provide a count_for_sitemap class method" if count_method.nil?
+        raise ArgumentError, "#{model} must provide a find_for_sitemap class method" if find_method.nil?
+
+        count        = model.send(count_method)
+        num_sitemaps = 1
+        num_batches  = 1
+
+        if count > @options[:batch_size]
+          num_batches  = (count.to_f / @options[:batch_size].to_f).ceil
+          num_sitemaps = (count.to_f / @options[:max_per_sitemap].to_f).ceil
+        end
+        batches_per_sitemap = num_batches.to_f / num_sitemaps.to_f
 
-        # Stream XML output to a file
-        filename = "sitemap_#{Extlib::Inflection::tableize(model.to_s)}"
-        filename << "_#{sitemap_num}" if num_sitemaps > 1
+        find_options = options.dup
 
-        f = xml_open(filename)
+        for sitemap_num in 1..num_sitemaps
+          # Work out the start and end batch numbers for this sitemap
+          batch_num_start = sitemap_num == 1 ? 1 : ((sitemap_num * batches_per_sitemap).ceil - batches_per_sitemap + 1).to_i
+          batch_num_end   = (batch_num_start + [batches_per_sitemap, num_batches].min).floor - 1
 
-        xml = Builder::XmlMarkup.new(:target => f)
-        xml.instruct!
-        xml.urlset(:xmlns => 'http://www.sitemaps.org/schemas/sitemap/0.9') do
           for batch_num in batch_num_start..batch_num_end
             offset       = ((batch_num - 1) * @options[:batch_size])
             limit        = (count - offset) < @options[:batch_size] ? (count - offset - 1) : @options[:batch_size]
             find_options.update(:limit => limit, :offset => offset) if num_batches > 1
 
-            model.send(find_method, find_options).each do |r|
-              last_mod_method = pick_method(r, TIMESTAMP_METHODS)
-              last_mod = last_mod_method.nil? ? Time.now : r.send(last_mod_method)
+            model.send(find_method, find_options).each do |record|
+              last_mod_method = pick_method(record, TIMESTAMP_METHODS)
+              last_mod = last_mod_method.nil? ? Time.now : record.send(last_mod_method)
 
-              param_method = pick_method(r, PARAM_METHODS)
+              param_method = pick_method(record, PARAM_METHODS)
 
-              xml.url do
-                location = defined?(Rails) ?
-                  polymorphic_url(r) :
-                  "#{root_url}/#{strip_leading_slash(options[:path])}/#{r.send(param_method)}"
-                xml.loc(location)
+              location = defined?(Rails) ?
+                polymorphic_url(record) :
+                "#{root_url}/#{strip_leading_slash(options[:path])}/#{record.send(param_method)}"
 
-                xml.lastmod(last_mod.strftime('%Y-%m-%d')) unless last_mod.nil?
+              change_frequency = options[:change_frequency] || 'weekly'
+              freq = change_frequency.is_a?(Proc) ? change_frequency.call(record) : change_frequency
 
-                change_frequency = options[:change_frequency] || 'weekly'
-                xml.changefreq(change_frequency.is_a?(Proc) ? change_frequency.call(r) : change_frequency)
+              priority = options[:priority]
+              pri = priority.is_a?(Proc) ? priority.call(record) : priority
 
-                priority = options[:priority]
-                unless priority.nil?
-                  xml.priority(priority.is_a?(Proc) ? priority.call(r) : priority)
-                end
-              end
+              sitemap.add_url!(location, last_mod, freq, pri)
             end
           end
         end
-
-        f.close
       end
-
     end
 
     generate_sitemap_index
@@ -193,6 +180,25 @@ class BigSitemap
 
   private
 
+  def with_sitemap(name, options={})
+    options[:index] = name == 'index'
+    options[:filename] = "#{@file_path}/sitemap_#{name}"
+    options[:max_urls] = @options[:max_per_sitemap]
+
+    unless options[:gzip] = @options[:gzip]
+      options[:indent] = 2
+    end
+
+    sitemap = Builder.new(options)
+
+    begin
+      yield sitemap
+    ensure
+      sitemap.close!
+      @sitemap_files.concat sitemap.paths!
+    end
+  end
+
   def strip_leading_slash(str)
     str.sub(/^\//, '')
   end
@@ -208,40 +214,15 @@ class BigSitemap
     method
   end
 
-  def xml_open(filename)
-    filename << '.xml'
-    filename << '.gz' if @options[:gzip]
-
-    file = File.open("#{@file_path}/#{filename}", 'w+')
-
-    @sitemap_files << file.path
-
-    writer = @options[:gzip] ? Zlib::GzipWriter.new(file) : file
-
-    if block_given?
-      yield writer
-      writer.close
-    end
-
-    writer
-  end
-
   def url_for_sitemap(path)
     "#{root_url}/#{File.basename(path)}"
   end
 
   # Create a sitemap index document
   def generate_sitemap_index
-    xml_open 'sitemap_index' do |file|
-      xml = Builder::XmlMarkup.new(:target => file)
-      xml.instruct!
-      xml.sitemapindex(:xmlns => 'http://www.sitemaps.org/schemas/sitemap/0.9') do
-        for path in @sitemap_files[0..-2]
-          xml.sitemap do
-            xml.loc(url_for_sitemap(path))
-            xml.lastmod(Time.now.strftime('%Y-%m-%d'))
-          end
-        end
+    with_sitemap 'index' do |sitemap|
+      for path in @sitemap_files
+        sitemap.add_url!(url_for_sitemap(path), File.stat(path).mtime)
       end
     end
   end

data/test/big_sitemap_test.rb

@@ -32,14 +32,13 @@ class BigSitemapTest < Test::Unit::TestCase
     create_sitemap
     add_model
     @sitemap.generate
-    assert File.exists?(single_sitemaps_model_file), "#{single_sitemaps_model_file} exists"
+    assert File.exists?(first_sitemaps_model_file), "#{first_sitemaps_model_file} exists"
   end
 
   should 'generate two sitemap model files' do
     generate_two_model_sitemap_files
     assert File.exists?(first_sitemaps_model_file), "#{first_sitemaps_model_file} exists"
     assert File.exists?(second_sitemaps_model_file), "#{second_sitemaps_model_file} exists"
-    third_sitemaps_model_file = "#{sitemaps_dir}/sitemap_test_model_3.xml.gz"
     assert !File.exists?(third_sitemaps_model_file), "#{third_sitemaps_model_file} does not exist"
   end
 
@@ -83,90 +82,90 @@ class BigSitemapTest < Test::Unit::TestCase
   context 'Sitemap model file' do
     should 'contain one urlset element' do
       generate_one_sitemap_model_file
-      assert_equal 1, num_elements(single_sitemaps_model_file, 'urlset')
+      assert_equal 1, num_elements(first_sitemaps_model_file, 'urlset')
     end
 
     should 'contain several loc elements' do
       generate_one_sitemap_model_file
-      assert_equal default_num_items, num_elements(single_sitemaps_model_file, 'loc')
+      assert_equal default_num_items, num_elements(first_sitemaps_model_file, 'loc')
     end
 
     should 'contain several lastmod elements' do
       generate_one_sitemap_model_file
-      assert_equal default_num_items, num_elements(single_sitemaps_model_file, 'lastmod')
+      assert_equal default_num_items, num_elements(first_sitemaps_model_file, 'lastmod')
     end
 
     should 'contain several changefreq elements' do
       generate_one_sitemap_model_file
-      assert_equal default_num_items, num_elements(single_sitemaps_model_file, 'changefreq')
+      assert_equal default_num_items, num_elements(first_sitemaps_model_file, 'changefreq')
     end
 
     should 'contain several priority elements' do
       generate_one_sitemap_model_file(:priority => 0.2)
-      assert_equal default_num_items, num_elements(single_sitemaps_model_file, 'priority')
+      assert_equal default_num_items, num_elements(first_sitemaps_model_file, 'priority')
     end
 
     should 'have a change frequency of weekly by default' do
       generate_one_sitemap_model_file
-      assert_equal 'weekly', elements(single_sitemaps_model_file, 'changefreq').first.text
+      assert_equal 'weekly', elements(first_sitemaps_model_file, 'changefreq').first.text
     end
 
     should 'have a change frequency of daily' do
       generate_one_sitemap_model_file(:change_frequency => 'daily')
-      assert_equal 'daily', elements(single_sitemaps_model_file, 'changefreq').first.text
+      assert_equal 'daily', elements(first_sitemaps_model_file, 'changefreq').first.text
     end
 
     should 'be able to use a lambda to specify change frequency' do
       generate_one_sitemap_model_file(:change_frequency => lambda {|m| m.change_frequency})
-      assert_equal TestModel.new.change_frequency, elements(single_sitemaps_model_file, 'changefreq').first.text
+      assert_equal TestModel.new.change_frequency, elements(first_sitemaps_model_file, 'changefreq').first.text
     end
 
     should 'have a priority of 0.2' do
       generate_one_sitemap_model_file(:priority => 0.2)
-      assert_equal '0.2', elements(single_sitemaps_model_file, 'priority').first.text
+      assert_equal '0.2', elements(first_sitemaps_model_file, 'priority').first.text
     end
 
     should 'be able to use a lambda to specify priority' do
       generate_one_sitemap_model_file(:priority => lambda {|m| m.priority})
-      assert_equal TestModel.new.priority.to_s, elements(single_sitemaps_model_file, 'priority').first.text
+      assert_equal TestModel.new.priority.to_s, elements(first_sitemaps_model_file, 'priority').first.text
     end
 
-    should 'contain one loc element' do
+    should 'contain two loc element' do
       generate_two_model_sitemap_files
-      assert_equal 1, num_elements(first_sitemaps_model_file, 'loc')
-      assert_equal 1, num_elements(second_sitemaps_model_file, 'loc')
+      assert_equal 2, num_elements(first_sitemaps_model_file, 'loc')
+      assert_equal 2, num_elements(second_sitemaps_model_file, 'loc')
     end
 
-    should 'contain one lastmod element' do
+    should 'contain two lastmod element' do
       generate_two_model_sitemap_files
-      assert_equal 1, num_elements(first_sitemaps_model_file, 'lastmod')
-      assert_equal 1, num_elements(second_sitemaps_model_file, 'lastmod')
+      assert_equal 2, num_elements(first_sitemaps_model_file, 'lastmod')
+      assert_equal 2, num_elements(second_sitemaps_model_file, 'lastmod')
     end
 
-    should 'contain one changefreq element' do
+    should 'contain two changefreq elements' do
       generate_two_model_sitemap_files
-      assert_equal 1, num_elements(first_sitemaps_model_file, 'changefreq')
-      assert_equal 1, num_elements(second_sitemaps_model_file, 'changefreq')
+      assert_equal 2, num_elements(first_sitemaps_model_file, 'changefreq')
+      assert_equal 2, num_elements(second_sitemaps_model_file, 'changefreq')
     end
 
-    should 'contain one priority element' do
+    should 'contain two priority element' do
       generate_two_model_sitemap_files(:priority => 0.2)
-      assert_equal 1, num_elements(first_sitemaps_model_file, 'priority')
-      assert_equal 1, num_elements(second_sitemaps_model_file, 'priority')
+      assert_equal 2, num_elements(first_sitemaps_model_file, 'priority')
+      assert_equal 2, num_elements(second_sitemaps_model_file, 'priority')
     end
 
     should 'strip leading slashes from controller paths' do
       create_sitemap
       @sitemap.add(TestModel, :path => '/test_controller').generate
       assert(
-        !elements(single_sitemaps_model_file, 'loc').first.text.match(/\/\/test_controller\//),
+        !elements(first_sitemaps_model_file, 'loc').first.text.match(/\/\/test_controller\//),
         'URL does not contain a double-slash before the controller path'
       )
     end
 
     should 'not be gzipped' do
       generate_one_sitemap_model_file(:gzip => false)
-      assert File.exists?(unzipped_single_sitemaps_model_file)
+      assert File.exists?(unzipped_first_sitemaps_model_file)
     end
   end
 
@@ -228,8 +227,8 @@ class BigSitemapTest < Test::Unit::TestCase
     def generate_two_model_sitemap_files(options={})
       change_frequency = options.delete(:change_frequency)
       priority         = options.delete(:priority)
-      create_sitemap(options.merge(:max_per_sitemap => 1, :batch_size => 1))
-      add_model(:num_items => 2, :change_frequency => change_frequency, :priority => priority)
+      create_sitemap(options.merge(:max_per_sitemap => 2, :batch_size => 1))
+      add_model(:num_items => 4, :change_frequency => change_frequency, :priority => priority)
       @sitemap.generate
     end
 
@@ -251,20 +250,20 @@ class BigSitemapTest < Test::Unit::TestCase
       "#{sitemaps_dir}/sitemap_index.xml"
     end
 
-    def single_sitemaps_model_file
-      "#{unzipped_single_sitemaps_model_file}.gz"
-    end
-
-    def unzipped_single_sitemaps_model_file
+    def unzipped_first_sitemaps_model_file
       "#{sitemaps_dir}/sitemap_test_models.xml"
     end
 
     def first_sitemaps_model_file
-      "#{sitemaps_dir}/sitemap_test_models_1.xml.gz"
+      "#{sitemaps_dir}/sitemap_test_models.xml.gz"
     end
 
     def second_sitemaps_model_file
-      "#{sitemaps_dir}/sitemap_test_models_2.xml.gz"
+      "#{sitemaps_dir}/sitemap_test_models_1.xml.gz"
+    end
+
+    def third_sitemaps_model_file
+      "#{sitemaps_dir}/sitemap_test_model_2.xml.gz"
     end
 
     def sitemaps_dir

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: alexrabarts-big_sitemap
 version: !ruby/object:Gem::Version 
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors: 
 - Alex Rabarts
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-04-06 00:00:00 -07:00
+date: 2009-04-18 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -45,6 +45,8 @@ files:
 - History.txt
 - README.rdoc
 - VERSION.yml
+- lib/big_sitemap
+- lib/big_sitemap/builder.rb
 - lib/big_sitemap.rb
 - test/big_sitemap_test.rb
 - test/fixtures