sitemap 0.1 → 0.2b

data/README.rdoc

@@ -43,10 +43,6 @@ You may change the defaults for either <tt>params</tt> or <tt>search</tt> option
 
   Sitemap.defaults[:params] = { :format => "html" }
 
-== Limitations
-
-Sitemaps can only have up to 50000 urls and an uncompressed size of 10MB. This issue will be resolved in a future release.
-
 == License
 
 This package is licensed under the MIT license and/or the Creative

data/lib/sitemap.rb

@@ -9,6 +9,7 @@ require "singleton"
 require "builder"
 require "sitemap/railtie"
 require "sitemap/ping"
+require "sitemap/store"
 require "sitemap/generator"
 
 module Sitemap
@@ -20,8 +21,12 @@ module Sitemap
   self.defaults = {
     :params => {},
     :search => {
-      :updated_at => proc { |obj| obj.updated_at.strftime("%Y-%m-%d") if obj.respond_to?(:updated_at) }
-    }
+      :updated_at => proc { |obj|
+        obj.updated_at.strftime("%Y-%m-%d") if obj.respond_to?(:updated_at)
+      }
+    },
+    :query_batch_size => 500,
+    :max_urls => 10000
   }
 
 end

data/lib/sitemap/generator.rb

@@ -10,13 +10,17 @@ module Sitemap
       :priority         => "priority"
     }
 
-    attr_accessor :entries, :host, :routes
+    attr_accessor :store, :host, :routes, :fragments
 
     # Instantiates a new object.
     # Should never be called directly.
     def initialize
       self.class.send(:include, Rails.application.routes.url_helpers)
-      self.entries = []
+      self.fragments = []
+      self.store = Store.new(:max_entries => Sitemap.defaults[:max_urls])
+      self.store.before_reset do |entries|
+        self.process_fragment!
+      end
     end
 
     # Sets the urls to be indexed.
@@ -75,7 +79,7 @@ module Sitemap
       search = Sitemap.defaults[:search].clone.merge!(options.select { |k, v| SEARCH_ATTRIBUTES.keys.include?(k) })
       search.merge!(search) { |type, value| get_data(object, value) }
 
-      self.entries << {
+      self.store << {
         :object => object,
         :search => search,
         :params => params
@@ -106,32 +110,65 @@ module Sitemap
     #
     def resources(type, options = {})
       path(type) unless options[:skip_index]
-      objects = options[:objects] ? options[:objects].call : type.to_s.classify.constantize.all
-      options.reject! { |k, v| k == :objects }
-
-      objects.each do |object|
-        path(object, options)
+      link_params = options.reject { |k, v| k == :objects }
+      get_objects = lambda {
+        options[:objects] ? options[:objects].call : type.to_s.classify.constantize
+      }
+      get_objects.call.find_each(:batch_size => Sitemap.defaults[:query_batch_size]) do |object|
+        path(object, link_params)
       end
     end
 
     # Parses the loaded data and returns the xml entries.
-    def build
-      instance_exec(self, &routes)
+    def render(object = "fragment")
       xml = Builder::XmlMarkup.new(:indent => 2)
-      file = File.read(File.expand_path("../../views/index.xml.builder", __FILE__))
+      file = File.read(File.expand_path("../../views/#{object}.xml.builder", __FILE__))
       instance_eval file
     end
 
-    # Builds xml entries and saves the data to the specified location.
-    def save(location)
-      file = File.new(location, "w")
-      file.write(build)
+    # Creates a temporary file from the existing entries.
+    def process_fragment!
+      file = Tempfile.new("sitemap.xml")
+      file.write(render)
       file.close
+      self.fragments << file
+    end
+
+    # Generates fragments.
+    def build!
+      instance_exec(self, &routes)
+      process_fragment! unless store.entries.empty?
+    end
+
+    # Creates the sitemap index file and saves any existing fragments.
+    def save(location)
+      if fragments.length == 1
+        FileUtils.mv(fragments.first.path, location)
+      else
+        remove_saved_files(location)
+        root = File.join(Pathname.new(location).dirname, "sitemap")
+        Dir.mkdir(root) unless File.directory?(root)
+        fragments.each_with_index do |fragment, i|
+          file_pattern = File.join(root, "sitemap-fragment-#{i + 1}.xml")
+          FileUtils.mv(fragment.path, file_pattern)
+        end
+        file = File.new(location, "w")
+        file.write(render "index")
+        file.close
+      end
+    end
+
+    # URL to the sitemap file.
+    #
+    # Defaults to <tt>sitemap.xml</tt>.
+    def file_url(path = "sitemap.xml")
+      URI::HTTP.build(:host => host, :path => File.join("/", path)).to_s
     end
 
-    # URL to the <tt>sitemap.xml</tt> file.
-    def file_url
-      URI::HTTP.build(:host => host, :path => "/sitemap.xml").to_s
+    def remove_saved_files(location)
+      root = File.join(Pathname.new(location).dirname, "sitemap")
+      Dir[File.join(root, "sitemap-fragment-*.xml")].each { |file| File.unlink(file) }
+      File.unlink(location) if File.exist?(location)
     end
 
     private

data/lib/sitemap/store.rb

@@ -0,0 +1,30 @@
+module Sitemap
+
+  class Store
+
+    attr_accessor :entries, :max_entries, :reset_count, :before_reset_callback
+
+    def initialize(options = {})
+      self.entries     = []
+      self.reset_count = 0
+      self.max_entries = options[:max_entries]
+    end
+
+    def << entry
+      reset! if entries.length >= max_entries
+      self.entries << entry
+    end
+
+    def reset!
+      before_reset_callback.call(entries) if before_reset_callback
+      self.entries = []
+      self.reset_count += 1
+    end
+
+    def before_reset(&block)
+      self.before_reset_callback = block
+    end
+
+  end
+
+end

data/lib/tasks/sitemap.rake

@@ -8,6 +8,7 @@ namespace :sitemap do
   task :generate => :environment do
     setup
     path = File.join(Rails.public_path, "sitemap.xml")
+    Sitemap::Generator.instance.build!
     Sitemap::Generator.instance.save path
   end
 

data/lib/views/fragment.xml.builder

@@ -0,0 +1,15 @@
+xml.instruct!
+xml.urlset :xmlns => "http://www.sitemaps.org/schemas/sitemap/0.9" do
+
+  store.entries.each do |entry|
+    xml.url do
+      xml.loc polymorphic_url(entry[:object], entry[:params])
+      entry[:search].each do |type, value|
+        next if !value || value.blank?
+        xml.tag! SEARCH_ATTRIBUTES[type], value.to_s
+      end
+    end
+  end
+
+end
+

data/lib/views/index.xml.builder

@@ -1,15 +1,11 @@
 xml.instruct!
-xml.urlset :xmlns => "http://www.sitemaps.org/schemas/sitemap/0.9" do
+xml.sitemapindex :xmlns => "http://www.sitemaps.org/schemas/sitemap/0.9" do
 
-  entries.each do |entry|
-    xml.url do
-      xml.loc polymorphic_url(entry[:object], entry[:params])
-      entry[:search].each do |type, value|
-        next if !value || value.blank?
-        xml.tag! SEARCH_ATTRIBUTES[type], value.to_s
-      end
+  fragments.each_with_index do |fragment, i|
+    xml.sitemap do
+      xml.loc file_url("sitemaps/sitemap-fragment-#{i + 1}.xml")
+      xml.lastmod Time.now.strftime("%Y-%m-%dT%H:%M:%S+00:00")
     end
   end
 
 end
-

data/sitemap.gemspec

@@ -2,7 +2,7 @@ $LOAD_PATH << File.join(File.dirname(__FILE__), "lib")
 
 spec = Gem::Specification.new do |spec|
   spec.name = "sitemap"
-  spec.version = "0.1"
+  spec.version = "0.2b"
   spec.summary = "Sitemap"
   spec.description = "A simple ruby on rails sitemap generator"
 

data/test/setup.rb

@@ -27,6 +27,7 @@ module SitemapTestSetup
         t.string :name
         t.text :contents
         t.string :location
+        t.boolean :published, :default => true
         t.timestamps
       end
     end
@@ -34,7 +35,8 @@ module SitemapTestSetup
       options = {
         :name => "Coding #{i}",
         :contents => "Lorem ipsum dolor sit",
-        :location => "someplace-#{i}"
+        :location => "someplace-#{i}",
+        :published => (i < 6)
       }
       Activity.create!(options)
     end

data/test/sitemap_test.rb

@@ -19,6 +19,7 @@ class SitemapTest < Test::Unit::TestCase
 
   def setup
     create_db
+    Sitemap.defaults[:max_urls] = 10000
     Sitemap::Generator.reset_instance
   end
 
@@ -28,7 +29,7 @@ class SitemapTest < Test::Unit::TestCase
 
   def test_xml_response
     Sitemap::Generator.instance.load(:host => "someplace.com") {}
-    doc = Nokogiri::XML(Sitemap::Generator.instance.build)
+    doc = Nokogiri::XML(Sitemap::Generator.instance.render)
     assert doc.errors.empty?
     assert_equal doc.root.name, "urlset"
   end
@@ -39,7 +40,8 @@ class SitemapTest < Test::Unit::TestCase
       path :root
       path :faq
     end
-    doc = Nokogiri::HTML(Sitemap::Generator.instance.build)
+    Sitemap::Generator.instance.build!
+    doc = Nokogiri::HTML(Sitemap::Generator.instance.render)
     elements = doc.xpath "//url/loc"
     assert_equal elements.length, urls.length
     elements.each_with_index do |element, i|
@@ -51,7 +53,8 @@ class SitemapTest < Test::Unit::TestCase
     Sitemap::Generator.instance.load(:host => "someplace.com") do
       resources :activities
     end
-    doc = Nokogiri::HTML(Sitemap::Generator.instance.build)
+    Sitemap::Generator.instance.build!
+    doc = Nokogiri::HTML(Sitemap::Generator.instance.render)
     elements = doc.xpath "//url/loc"
     assert_equal elements.length, Activity.count + 1
     assert_equal elements.first.text, "http://someplace.com/activities"
@@ -61,14 +64,15 @@ class SitemapTest < Test::Unit::TestCase
   end
 
   def test_custom_resource_objects
-    activities = [Activity.first, Activity.last]
+    activities = proc { Activity.where(:published => true) }
     Sitemap::Generator.instance.load(:host => "someplace.com") do
-      resources :activities, :objects => proc { activities }, :skip_index => true
+      resources :activities, :objects => activities, :skip_index => true
     end
-    doc = Nokogiri::HTML(Sitemap::Generator.instance.build)
+    Sitemap::Generator.instance.build!
+    doc = Nokogiri::HTML(Sitemap::Generator.instance.render)
     elements = doc.xpath "//url/loc"
-    assert_equal elements.length, activities.length
-    activities.each_with_index do |activity, i|
+    assert_equal elements.length, activities.call.length
+    activities.call.each_with_index do |activity, i|
       assert_equal elements[i].text, "http://someplace.com/activities/%d" % activity.id
     end
   end
@@ -77,7 +81,8 @@ class SitemapTest < Test::Unit::TestCase
     Sitemap::Generator.instance.load(:host => "someplace.com") do
       path :faq, :params => { :host => "anotherplace.com", :format => "html", :filter => "recent" }
     end
-    doc = Nokogiri::HTML(Sitemap::Generator.instance.build)
+    Sitemap::Generator.instance.build!
+    doc = Nokogiri::HTML(Sitemap::Generator.instance.render)
     elements = doc.xpath "//url/loc"
     assert_equal elements.first.text, "http://anotherplace.com/questions.html?filter=recent"
   end
@@ -87,7 +92,8 @@ class SitemapTest < Test::Unit::TestCase
       resources :activities, :skip_index => true, :params => { :host => proc { |obj| [obj.location, host].join(".") } }
     end
     activities = Activity.all
-    doc = Nokogiri::HTML(Sitemap::Generator.instance.build)
+    Sitemap::Generator.instance.build!
+    doc = Nokogiri::HTML(Sitemap::Generator.instance.render)
     elements = doc.xpath "//url/loc"
     elements.each_with_index do |element, i|
       assert_equal element.text, "http://%s.someplace.com/activities/%d" % [activities[i].location, activities[i].id]
@@ -99,7 +105,8 @@ class SitemapTest < Test::Unit::TestCase
       path :faq, :priority => 1, :change_frequency => "always"
       resources :activities, :change_frequency => "weekly"
     end
-    doc = Nokogiri::HTML(Sitemap::Generator.instance.build)
+    Sitemap::Generator.instance.build!
+    doc = Nokogiri::HTML(Sitemap::Generator.instance.render)
     assert_equal doc.xpath("//url/priority").first.text, "1"
     elements = doc.xpath "//url/changefreq"
     assert_equal elements[0].text, "always"
@@ -113,7 +120,7 @@ class SitemapTest < Test::Unit::TestCase
       resources :activities, :priority => proc { |obj| obj.id <= 2 ? 1 : 0.5 }, :skip_index => true
     end
     activities = Activity.all
-    doc = Nokogiri::HTML(Sitemap::Generator.instance.build)
+    doc = Nokogiri::HTML(Sitemap::Generator.instance.render)
     elements = doc.xpath "//url/priority"
     elements.each_with_index do |element, i|
       value = activities[i].id <= 2 ? "1" : "0.5"
@@ -125,7 +132,8 @@ class SitemapTest < Test::Unit::TestCase
     Sitemap::Generator.instance.load(:host => "someplace.com") do
       path :faq, :priority => "", :change_frequency => lambda { |e| return false}, :updated_at => Date.today
     end
-    doc = Nokogiri::HTML(Sitemap::Generator.instance.build)
+    Sitemap::Generator.instance.build!
+    doc = Nokogiri::HTML(Sitemap::Generator.instance.render)
     assert_equal doc.xpath("//url/priority").count, 0
     assert_equal doc.xpath("//url/changefreq").count, 0
     assert_equal doc.xpath("//url/lastmod").text, Date.today.to_s
@@ -136,4 +144,50 @@ class SitemapTest < Test::Unit::TestCase
     assert_equal Sitemap::Generator.instance.file_url, "http://someplace.com/sitemap.xml"
   end
 
+  def test_save_creates_file
+    path = File.join(Dir.tmpdir, "sitemap.xml")
+    File.unlink(path) if File.exist?(path)
+    Sitemap::Generator.instance.load(:host => "someplace.com") do
+      resources :activities
+    end
+    Sitemap::Generator.instance.build!
+    Sitemap::Generator.instance.save(path)
+    assert File.exist?(path)
+    File.unlink(path)
+  end
+
+  def test_saves_fragments
+    Sitemap.defaults[:max_urls] = 2
+    Sitemap::Generator.instance.load(:host => "someplace.com") do
+      path :root
+      path :root
+      path :root
+      path :root
+    end
+    path = File.join(Dir.tmpdir, "sitemap.xml")
+    root = File.join(Dir.tmpdir, "sitemap") # Directory is being removed at the end of the test.
+    assert !File.directory?(root)
+    Sitemap::Generator.instance.build!
+    Sitemap::Generator.instance.save(path)
+    1.upto(2) { |i|
+      assert File.exists?(File.join(root, "sitemap-fragment-#{i}.xml"))
+    }
+    FileUtils.rm_rf(root)
+  end
+
+  def test_fragments_index
+    Sitemap.defaults[:max_urls] = 2
+    Sitemap::Generator.instance.load(:host => "someplace.com") do
+      path :root
+      path :root
+      path :root
+      path :root
+      path :root
+    end
+    Sitemap::Generator.instance.build!
+    doc = Nokogiri::HTML(Sitemap::Generator.instance.render("index"))
+    elements = doc.xpath "//sitemap"
+    assert_equal Sitemap::Generator.instance.fragments.length, 3
+  end
+
 end

data/test/store_test.rb

@@ -0,0 +1,35 @@
+require "test/unit"
+require "rubygems"
+
+class StoreTest < Test::Unit::TestCase
+
+  def test_append_entries
+    store = Sitemap::Store.new(:max_entries => 1000)
+    3.times { store << "contents" }
+    assert_equal store.entries.length, 3
+  end
+
+  def test_reset_entries_limit
+    store = Sitemap::Store.new(:max_entries => 2)
+    2.times { store << "contents" }
+    assert_equal store.entries.length, 2
+    store << "contents"
+    assert_equal store.entries.length, 1
+  end
+
+  def test_reset_callback
+    store = Sitemap::Store.new(:max_entries => 2)
+    store.before_reset do |entries|
+      store.instance_variable_set("@callback_data", entries.join(", "))
+    end
+    3.times { |i| store << "item #{i + 1}" }
+    assert_equal store.instance_variable_get("@callback_data"), "item 1, item 2"
+  end
+
+  def test_increments_reset_count
+    store = Sitemap::Store.new(:max_entries => 2)
+    5.times { store << "contents" }
+    assert_equal store.reset_count, 2
+  end
+
+end

metadata

@@ -1,19 +1,19 @@
 --- !ruby/object:Gem::Specification
 name: sitemap
 version: !ruby/object:Gem::Version
-  version: '0.1'
-  prerelease: 
+  version: 0.2b
+  prerelease: 3
 platform: ruby
 authors:
 - Daniel Mircea
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-09-10 00:00:00.000000000Z
+date: 2012-03-02 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sqlite3
-  requirement: &17823580 !ruby/object:Gem::Requirement
+  requirement: &13114780 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *17823580
+  version_requirements: *13114780
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &17823140 !ruby/object:Gem::Requirement
+  requirement: &13114300 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,7 +32,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *17823140
+  version_requirements: *13114300
 description: A simple ruby on rails sitemap generator
 email: daniel@viseztrance.com
 executables: []
@@ -41,21 +41,24 @@ extra_rdoc_files:
 - README.rdoc
 - LICENSE
 files:
-- lib/generators/sitemap/USAGE
-- lib/generators/sitemap/templates/sitemap.rb
-- lib/generators/sitemap/install_generator.rb
-- lib/views/index.xml.builder
-- lib/sitemap.rb
+- lib/tasks/sitemap.rake
+- lib/sitemap/ping.rb
 - lib/sitemap/railtie.rb
+- lib/sitemap/store.rb
 - lib/sitemap/generator.rb
-- lib/sitemap/ping.rb
-- lib/tasks/sitemap.rake
+- lib/views/index.xml.builder
+- lib/views/fragment.xml.builder
+- lib/sitemap.rb
+- lib/generators/sitemap/USAGE
+- lib/generators/sitemap/install_generator.rb
+- lib/generators/sitemap/templates/sitemap.rb
 - README.rdoc
 - LICENSE
 - Rakefile
 - sitemap.gemspec
 - test/singleton.rb
 - test/setup.rb
+- test/store_test.rb
 - test/sitemap_test.rb
 homepage: http://github.com/viseztrance/rails-sitemap
 licenses: []
@@ -77,16 +80,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
-  - - ! '>='
+  - - ! '>'
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.8
+rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
 summary: Sitemap
 test_files:
 - test/singleton.rb
 - test/setup.rb
+- test/store_test.rb
 - test/sitemap_test.rb
+has_rdoc: true