sitemap_generator 0.2.6 → 0.3.0

data/README.md

@@ -192,7 +192,6 @@ Notes
 Known Bugs
 ========
 
-- Sitemaps.org [states][sitemaps_org] that no Sitemap XML file should be more than 10Mb uncompressed. The plugin will warn you about this, but does nothing to avoid it (like move some URLs into a later file).
 - There's no check on the size of a URL which [isn't supposed to exceed 2,048 bytes][sitemaps_xml].
 - Currently only supports one Sitemap Index file, which can contain 50,000 Sitemap files which can each contain 50,000 urls, so it _only_ supports up to 2,500,000,000 (2.5 billion) urls. I personally have no need of support for more urls, but plugin could be improved to support this.
 

data/Rakefile

@@ -22,6 +22,28 @@ rescue LoadError
   puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
 end
 
+#
+# Helper methods
+#
+module Helpers
+  extend self
+
+  # Return a full local path to path fragment <tt>path</tt>
+  def local_path(path)
+    File.join(File.dirname(__FILE__), path)
+  end
+
+  # Copy all of the local files into <tt>path</tt> after completely cleaning it
+  def prepare_path(path)
+    rm_rf path
+    mkdir_p path
+    cp_r(FileList["[A-Z]*", "{bin,lib,rails,templates,tasks}"], path)
+  end
+end
+
+#
+# Tasks
+#
 task :default => :test
 
 namespace :test do
@@ -37,29 +59,19 @@ namespace :test do
   namespace :prepare do
     task :gem do
       ENV["SITEMAP_RAILS"] = 'gem'
-      prepare_path(local_path('spec/mock_app_gem/vendor/gems/sitemap_generator-1.2.3'))
-      rm_rf(local_path('spec/mock_app_gem/public/sitemap*'))
+      Helpers.prepare_path(Helpers.local_path('spec/mock_app_gem/vendor/gems/sitemap_generator-1.2.3'))
+      rm_rf(Helpers.local_path('spec/mock_app_gem/public/sitemap*'))
     end
 
     task :plugin do
       ENV["SITEMAP_RAILS"] = 'plugin'
-      prepare_path(local_path('spec/mock_app_plugin/vendor/plugins/sitemap_generator-1.2.3'))
-      rm_rf(local_path('spec/mock_app_plugin/public/sitemap*'))
+      Helpers.prepare_path(Helpers.local_path('spec/mock_app_plugin/vendor/plugins/sitemap_generator-1.2.3'))
+      rm_rf(Helpers.local_path('spec/mock_app_plugin/public/sitemap*'))
     end
 
     task :rails3 do
       ENV["SITEMAP_RAILS"] = 'rails3'
-      rm_rf(local_path('spec/mock_rails3_gem/public/sitemap*'))
-    end
-
-    def local_path(path)
-      File.join(File.dirname(__FILE__), path)
-    end
-
-    def prepare_path(path)
-      rm_rf path
-      mkdir_p path
-      cp_r(FileList["[A-Z]*", "{bin,lib,rails,templates,tasks}"], path)
+      rm_rf(Helpers.local_path('spec/mock_rails3_gem/public/sitemap*'))
     end
   end
 end
@@ -86,4 +98,4 @@ Rake::RDocTask.new(:rdoc) do |rdoc|
   rdoc.options << '--line-numbers' << '--inline-source'
   rdoc.rdoc_files.include('README.md')
   rdoc.rdoc_files.include('lib/**/*.rb')
-end
+end

data/VERSION

@@ -1 +1 @@
-0.2.6
+0.3.0

data/lib/sitemap_generator/builder/helper.rb

@@ -0,0 +1,10 @@
+module SitemapGenerator
+  module Builder
+    module Helper
+
+      def w3c_date(date)
+         date.utc.strftime("%Y-%m-%dT%H:%M:%S+00:00")
+      end
+    end
+  end
+end

data/lib/sitemap_generator/builder/sitemap_file.rb

@@ -0,0 +1,114 @@
+require 'sitemap_generator/builder/helper'
+require 'builder'
+require 'zlib'
+
+module SitemapGenerator
+  module Builder
+    class SitemapFile
+      include SitemapGenerator::Builder::Helper
+
+      attr_accessor :sitemap_path, :public_path, :filesize, :link_count, :hostname
+
+      # <tt>public_path</tt> full path of the directory to write sitemaps in.
+      #   Usually your Rails <tt>public/</tt> directory.
+      #
+      # <tt>sitemap_path</tt> relative path including filename of the sitemap
+      #   file relative to <tt>public_path</tt>
+      #
+      # <tt>hostname</tt> hostname including protocol to use in all links
+      #   e.g. http://en.google.ca
+      def initialize(public_path, sitemap_path, hostname)
+        self.sitemap_path = sitemap_path
+        self.public_path = public_path
+        self.hostname = hostname
+        self.link_count = 0
+
+        @xml_content       = ''     # XML urlset content
+        @xml_wrapper_start = %q[<?xml version="1.0" encoding="UTF-8"?><urlset xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/siteindex.xsd" xmlns:image="http://www.google.com/schemas/sitemap-image/1.1" xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">]
+        @xml_wrapper_end   = %q[</urlset>]
+        self.filesize = @xml_wrapper_start.bytesize + @xml_wrapper_end.bytesize
+      end
+
+      def lastmod
+        File.mtime(self.full_path) rescue nil
+      end
+
+      def empty?
+        self.link_count == 0
+      end
+
+      def full_url
+        URI.join(self.hostname, self.sitemap_path).to_s
+      end
+
+      def full_path
+        @full_path ||= File.join(self.public_path, self.sitemap_path)
+      end
+
+      # Return a boolean indicating whether the sitemap file can fit another link
+      # of <tt>bytes</tt> bytes in size.
+      def file_can_fit?(bytes)
+        (self.filesize + bytes) < SitemapGenerator::MAX_SITEMAP_FILESIZE && self.link_count < SitemapGenerator::MAX_SITEMAP_LINKS
+      end
+
+      # Add a link to the sitemap file and return a boolean indicating whether the
+      # link was added.
+      #
+      # If a link cannot be added, the file is too large or the link limit has been reached.
+      def add_link(link)
+        xml = build_xml(::Builder::XmlMarkup.new, link)
+        unless file_can_fit?(xml.bytesize)
+          self.finalize!
+          return false
+        end
+
+        @xml_content << xml
+        self.filesize += xml.bytesize
+        self.link_count += 1
+        true
+      end
+      alias_method :<<, :add_link
+
+      # Return XML as a String
+      def build_xml(builder, link)
+        builder.url do
+          builder.loc        link[:loc]
+          builder.lastmod    w3c_date(link[:lastmod])   if link[:lastmod]
+          builder.changefreq link[:changefreq]          if link[:changefreq]
+          builder.priority   link[:priority]            if link[:priority]
+
+          unless link[:images].blank?
+            link[:images].each do |image|
+              builder.image:image do
+                builder.image :loc, image[:loc]
+                builder.image :caption, image[:caption]             if image[:caption]
+                builder.image :geo_location, image[:geo_location]   if image[:geo_location]
+                builder.image :title, image[:title]                 if image[:title]
+                builder.image :license, image[:license]             if image[:license]
+              end
+            end
+          end
+        end
+        builder << ''
+      end
+
+      # Insert the content into the XML "wrapper" and write and close the file.
+      #
+      # All the xml content in the instance is cleared, but attributes like
+      # <tt>filesize</tt> are still available.
+      def finalize!
+        return if self.frozen?
+
+        open(self.full_path, 'w') do |file|
+          gz = Zlib::GzipWriter.new(file)
+          gz.write @xml_wrapper_start
+          gz.write @xml_content
+          gz.write @xml_wrapper_end
+          gz.close
+        end
+        @xml_content = @xml_wrapper_start = @xml_wrapper_end = ''
+        self.freeze
+      end
+    end
+  end
+end

data/lib/sitemap_generator/builder/sitemap_index_file.rb

@@ -0,0 +1,24 @@
+module SitemapGenerator
+  module Builder
+    class SitemapIndexFile < SitemapFile
+
+      def initialize(*args)
+        super(*args)
+
+        @ml_content = ''     # XML urlset content
+        @xml_wrapper_start = %q[<?xml version="1.0" encoding="UTF-8"?><sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">]
+        @xml_wrapper_end   = %q[</sitemapindex>]
+        self.filesize = @xml_wrapper_start.bytesize + @xml_wrapper_end.bytesize
+      end
+
+      # Return XML as a String
+      def build_xml(builder, link)
+        builder.url do
+          builder.loc        link[:loc]
+          builder.lastmod    w3c_date(link[:lastmod])   if link[:lastmod]
+        end
+        builder << ''
+      end
+    end
+  end
+end

data/lib/sitemap_generator/builder.rb

@@ -0,0 +1,9 @@
+require 'sitemap_generator/builder/helper'
+require 'sitemap_generator/builder/sitemap_file'
+require 'sitemap_generator/builder/sitemap_index_file'
+
+module SitemapGenerator
+  module Builder
+
+  end
+end

data/lib/sitemap_generator/interpreter.rb

@@ -0,0 +1,28 @@
+module SitemapGenerator
+
+  # Evaluate a sitemap config file within the context of a class that includes the
+  # Rails URL helpers.
+  class Interpreter
+
+    if SitemapGenerator::Utilities.rails3?
+      include ::Rails.application.routes.url_helpers
+    else
+      require 'action_controller'
+      include ActionController::UrlWriter
+    end
+
+    def initialize(sitemap_config_file=nil)
+      sitemap_config_file ||= File.join(::Rails.root, 'config/sitemap.rb')
+      eval(open(sitemap_config_file).read)
+    end
+
+    # KJV do we need this?  We should be using path_* helpers.
+    # def self.default_url_options(options = nil)
+    #   { :host => SitemapGenerator::Sitemap.default_host }
+    # end
+
+    def self.run
+      new
+    end
+  end
+end

data/lib/sitemap_generator/link.rb

@@ -1,30 +1,35 @@
 module SitemapGenerator
-  class Link
-    class << self
-      def generate(path, options = {})
-        options.assert_valid_keys(:priority, :changefreq, :lastmod, :host, :images)
-        options.reverse_merge!(:priority => 0.5, :changefreq => 'weekly', :lastmod => Time.now, :host => Sitemap.default_host, :images => [])
-        {
-          :path => path,
-          :priority => options[:priority],
-          :changefreq => options[:changefreq],
-          :lastmod => options[:lastmod],
-          :host => options[:host],
-          :loc => URI.join(options[:host], path).to_s,
-          :images => prepare_images(options[:images], options[:host])
-        }
+  module Link
+    extend self
+
+    # Return a Hash of options suitable to pass to a SitemapGenerator::Builder::SitemapFile instance.
+    def generate(path, options = {})
+      if path.is_a?(SitemapGenerator::Builder::SitemapFile)
+        options.reverse_merge!(:host => path.hostname, :lastmod => path.lastmod)
+        path = path.sitemap_path
       end
 
-      # Maximum 1000 images.  <tt>loc</tt> is required.
-      # ?? Does the image URL have to be on the same host?
-      def prepare_images(images, host)
-        images.delete_if { |key,value| key[:loc] == nil }
-        images.each do |r|
-          r.assert_valid_keys(:loc, :caption, :geo_location, :title, :license)
-          r[:loc] = URI.join(host, r[:loc]).to_s
-        end
-        images[0..(SitemapGenerator::MAX_IMAGES-1)]
+      options.assert_valid_keys(:priority, :changefreq, :lastmod, :host, :images)
+      options.reverse_merge!(:priority => 0.5, :changefreq => 'weekly', :lastmod => Time.now, :host => Sitemap.default_host, :images => [])
+      {
+        :path => path,
+        :priority => options[:priority],
+        :changefreq => options[:changefreq],
+        :lastmod => options[:lastmod],
+        :host => options[:host],
+        :loc => URI.join(options[:host], path).to_s,
+        :images => prepare_images(options[:images], options[:host])
+      }
+    end
+
+    # Return an Array of image option Hashes suitable to be parsed by SitemapGenerator::Builder::SitemapFile
+    def prepare_images(images, host)
+      images.delete_if { |key,value| key[:loc] == nil }
+      images.each do |r|
+        r.assert_valid_keys(:loc, :caption, :geo_location, :title, :license)
+        r[:loc] = URI.join(host, r[:loc]).to_s
       end
+      images[0..(SitemapGenerator::MAX_SITEMAP_IMAGES-1)]
     end
   end
 end

data/lib/sitemap_generator/link_set.rb

@@ -1,166 +1,173 @@
 require 'builder'
 require 'action_view'
 
+# A LinkSet provisions a bunch of links to sitemap files.  It also writes the index file
+# which lists all the sitemap files written.
 module SitemapGenerator
   class LinkSet
-    include SitemapGenerator::Helper
-    include ActionView::Helpers::NumberHelper
-
-    attr_accessor :default_host, :yahoo_app_id, :links
-    attr_accessor :sitemaps
-    attr_accessor :max_entries
-    attr_accessor :link_count
-
-    alias :sitemap_files :sitemaps
-
-    # Create new link set instance.
-    def initialize
-      self.links       = []
-      self.sitemaps    = []
-      self.max_entries = SitemapGenerator::MAX_ENTRIES
-      self.link_count  = 0
-    end
+    include ActionView::Helpers::NumberHelper  # for number_with_delimiter
 
-    # Add default links to sitemap files.
-    def add_default_links
-      links.push Link.generate('/', :lastmod => Time.now, :changefreq => 'always', :priority => 1.0)
-      links.push Link.generate("/#{index_file}", :lastmod => Time.now, :changefreq => 'always', :priority => 1.0)
-      self.link_count += 2
-    end
+    attr_accessor :default_host, :public_path, :sitemaps_path
+    attr_accessor :sitemap, :sitemaps, :sitemap_index
+    attr_accessor :verbose, :yahoo_app_id
 
-    # Add links to sitemap files passing a block.
-    def add_links
-      raise ArgumentError, "Default hostname not set" if default_host.blank?
-      add_default_links if first_link?
-      yield Mapper.new(self)
-    end
+    # Evaluate the sitemap config file and write all sitemaps.
+    #
+    # This should be refactored so that we can have multiple instances
+    # of LinkSet.
+    def create
+      require 'sitemap_generator/interpreter'
 
-    # Add links from mapper to sitemap files.
-    def add_link(link)
-      write_upcoming if enough_links?
-      links.push link
-      self.link_count += 1
-    end
+      start_time = Time.now
+      SitemapGenerator::Interpreter.run
+      finalize!
+      end_time = Time.now
 
-    # Write links to sitemap file.
-    def write
-      write_pending
+      puts "\nSitemap stats: #{number_with_delimiter(self.link_count)} links / #{self.sitemaps.size} files / " + ("%dm%02ds" % (end_time - start_time).divmod(60)) if verbose
     end
 
-    # Write links to upcoming sitemap file.
-    def write_upcoming
-      write_sitemap(upcoming_file)
+    # <tt>public_path</tt> (optional) full path to the directory to write sitemaps in.
+    #   Defaults to your Rails <tt>public/</tt> directory.
+    #
+    # <tt>sitemaps_path</tt> (optional) path fragment within public to write sitemaps
+    #   to e.g. 'en/'.  Sitemaps are written to <tt>public_path</tt> + <tt>sitemaps_path</tt>
+    #
+    # <tt>default_host</tt> hostname including protocol to use in all sitemap links
+    #   e.g. http://en.google.ca
+    def initialize(public_path = nil, sitemaps_path = nil, default_host = nil)
+      public_path = File.join(::Rails.root, 'public/') if public_path.nil?
+      self.default_host = default_host
+      self.public_path = public_path
+      self.sitemaps_path = sitemaps_path
+
+      # Completed sitemaps
+      self.sitemaps = []
     end
 
-    # Write pending links to sitemap, write index file if needed.
-    def write_pending
-      write_upcoming
-      write_index
+    def link_count
+      self.sitemaps.map(&:link_count).inject(:+)
     end
 
-    # Write links to sitemap file.
-    def write_sitemap(file = upcoming_file)
-      slice_index = 0
-      buffer = ""
-      xml = Builder::XmlMarkup.new(:target => buffer)
-      eval(SitemapGenerator.templates.sitemap_xml, binding)
-      filename = File.join(Rails.root, "public", file)
-      write_file(filename, buffer)
-      show_progress("Sitemap", filename, buffer) if verbose
-      if slice_index==0
-        links.clear
-      else
-        links.slice! slice_index, links.size
-      end
+    # Called within the user's eval'ed sitemap config file.  Add links to sitemap files
+    # passing a block.
+    #
+    # TODO: Refactor.  The call chain is confusing and convoluted here.
+    def add_links
+      raise ArgumentError, "Default hostname not set" if default_host.blank?
 
-      sitemaps.push filename
-    end
+      # I'd rather have these calls in <tt>create</tt> but we have to wait
+      # for <tt>default_host</tt> to be set by the user's sitemap config
+      new_sitemap
+      add_default_links
 
-    # Write sitemap links to sitemap index file.
-    def write_index
-      buffer = ""
-      xml = Builder::XmlMarkup.new(:target => buffer)
-      eval(SitemapGenerator.templates.sitemap_index, binding)
-      filename = File.join(Rails.root, "public", index_file)
-      write_file(filename, buffer)
-      show_progress("Sitemap Index", filename, buffer) if verbose
-      links.clear
-      sitemaps.clear
+      yield Mapper.new(self)
     end
 
-    # Return sitemap or sitemap index main name.
-    def index_file
-      "sitemap_index.xml.gz"
+    # Called from Mapper.
+    #
+    # Add a link to the current sitemap.
+    def add_link(link)
+      unless self.sitemap << link
+        new_sitemap
+        self.sitemap << link
+      end
     end
 
-    # Return upcoming sitemap name with index.
-    def upcoming_file
-      "sitemap#{upcoming_index}.xml.gz" unless enough_sitemaps?
-    end
+    # Add the current sitemap to the <tt>sitemaps</tt> Array and
+    # start a new sitemap.
+    #
+    # If the current sitemap is nil or empty it is not added.
+    def new_sitemap
+      unless self.sitemap_index
+        self.sitemap_index = SitemapGenerator::Builder::SitemapIndexFile.new(public_path, sitemap_index_path, default_host)
+      end
 
-    # Return upcoming sitemap index, first is 1.
-    def upcoming_index
-      sitemaps.length + 1 unless enough_sitemaps?
-    end
+      unless self.sitemap
+        self.sitemap = SitemapGenerator::Builder::SitemapFile.new(public_path, new_sitemap_path, default_host)
+      end
 
-    # Return true if upcoming is first sitemap.
-    def first_sitemap?
-      sitemaps.empty?
-    end
+      # Mark the sitemap as complete and add it to the sitemap index
+      unless self.sitemap.empty?
+        self.sitemap.finalize!
+        self.sitemap_index << Link.generate(self.sitemap)
+        self.sitemaps << self.sitemap
+        show_progress(self.sitemap) if verbose
 
-    # Return true if sitemap index needed.
-    def multiple_sitemaps?
-      !first_sitemap?
+        self.sitemap = SitemapGenerator::Builder::SitemapFile.new(public_path, new_sitemap_path, default_host)
+      end
     end
 
-    # Return true if more sitemaps can be added.
-    def more_sitemaps?
-      sitemaps.length < max_entries
+    # Report progress line.
+    def show_progress(sitemap)
+      uncompressed_size = number_to_human_size(sitemap.filesize)
+      compressed_size =   number_to_human_size(File.size?(sitemap.full_path))
+      puts "+ #{sitemap.sitemap_path}   #{sitemap.link_count} links / #{uncompressed_size} / #{compressed_size} gzipped"
     end
 
-    # Return true if no sitemaps can be added.
-    def enough_sitemaps?
-      !more_sitemaps?
+    # Finalize all sitemap files
+    def finalize!
+      new_sitemap
+      self.sitemap_index.finalize!
     end
 
-    # Return true if this is the first link added.
-    def first_link?
-      links.empty? && first_sitemap?
-    end
+    # Ping search engines.
+    #
+    # @see http://en.wikipedia.org/wiki/Sitemap_index
+    def ping_search_engines
+      require 'open-uri'
+
+      sitemap_index_url = CGI.escape(self.sitemap_index.full_url)
+      search_engines = {
+        :google         => "http://www.google.com/webmasters/sitemaps/ping?sitemap=#{sitemap_index_url}",
+        :yahoo          => "http://search.yahooapis.com/SiteExplorerService/V1/ping?sitemap=#{sitemap_index_url}&appid=#{yahoo_app_id}",
+        :ask            => "http://submissions.ask.com/ping?sitemap=#{sitemap_index_url}",
+        :bing           => "http://www.bing.com/webmaster/ping.aspx?siteMap=#{sitemap_index_url}",
+        :sitemap_writer => "http://www.sitemapwriter.com/notify.php?crawler=all&url=#{sitemap_index_url}"
+      }
+
+      puts "\n" if verbose
+      search_engines.each do |engine, link|
+        next if engine == :yahoo && !self.yahoo_app_id
+        begin
+          open(link)
+          puts "Successful ping of #{engine.to_s.titleize}" if verbose
+        rescue Timeout::Error, StandardError => e
+          puts "Ping failed for #{engine.to_s.titleize}: #{e.inspect} (URL #{link})" if verbose
+        end
+      end
 
-    # Return true if more links can be added.
-    def more_links?
-      links.length < max_entries
-    end
+      if !self.yahoo_app_id && verbose
+        puts "\n"
+        puts <<-END.gsub(/^\s+/, '')
+          To ping Yahoo you require a Yahoo AppID.  Add it to your config/sitemap.rb with:
 
-    # Return true if no further links can be added.
-    def enough_links?
-      !more_links?
-    end
+          SitemapGenerator::Sitemap.yahoo_app_id = "my_app_id"
 
-    # Commit buffer to gzipped file.
-    def write_file(name, buffer)
-      Zlib::GzipWriter.open(name) { |gz| gz.write buffer }
+          For more information see http://developer.yahoo.com/search/siteexplorer/V1/updateNotification.html
+        END
+      end
     end
 
-    # Report progress line.
-    def show_progress(title, filename, buffer)
-      puts "+ #{filename}"
-      puts "** #{title} too big! The uncompressed size exceeds 10Mb" if buffer.size > 10.megabytes
+    protected
+
+    def add_default_links
+      self.sitemap << Link.generate('/', :lastmod => Time.now, :changefreq => 'always', :priority => 1.0)
+      self.sitemap << Link.generate(self.sitemap_index, :lastmod => Time.now, :changefreq => 'always', :priority => 1.0)
     end
 
-    # Ping search engines passing sitemap location.
-    def ping_search_engines
-      super index_file
+    # Return the current sitemap filename with index.
+    #
+    # The index depends on the length of the <tt>sitemaps</tt> array.
+    def new_sitemap_path
+      File.join(self.sitemaps_path || '', "sitemap#{self.sitemaps.length + 1}.xml.gz")
     end
 
-    # Create sitemap files in output directory.
-    def create_files(verbose = true)
-      start_time = Time.now
-      load_sitemap_rb
-      write
-      stop_time = Time.now
-      puts "Sitemap stats: #{number_with_delimiter(SitemapGenerator::Sitemap.link_count)} links, " + ("%dm%02ds" % (stop_time - start_time).divmod(60)) if verbose
+    # Return the current sitemap index filename.
+    #
+    # At the moment we only support one index file which can link to
+    # up to 50,000 sitemap files.
+    def sitemap_index_path
+      File.join(self.sitemaps_path || '', 'sitemap_index.xml.gz')
     end
   end
-end
+end

data/lib/sitemap_generator/templates.rb

@@ -7,8 +7,6 @@ module SitemapGenerator
   # Define an accessor method for each template file.
   class Templates
     FILES = {
-      :sitemap_index  =>  'sitemap_index.builder',
-      :sitemap_xml    =>  'xml_sitemap.builder',
       :sitemap_sample =>  'sitemap.rb',
     }
 
@@ -28,7 +26,7 @@ module SitemapGenerator
 
     # Return the full path to a template.
     #
-    # <tt>file</tt> template symbol e.g. <tt>:sitemap_index</tt>
+    # <tt>file</tt> template symbol e.g. <tt>:sitemap_sample</tt>
     def template_path(template)
       File.join(@root, 'templates', self.class::FILES[template])
     end

data/lib/sitemap_generator/utilities.rb

@@ -3,14 +3,14 @@ module SitemapGenerator
     extend self
 
     # Copy templates/sitemap.rb to config if not there yet.
-    def install_sitemap_rb
+    def install_sitemap_rb(verbose=false)
       if File.exist?(File.join(RAILS_ROOT, 'config/sitemap.rb'))
-        puts "already exists: config/sitemap.rb, file not copied"
+        puts "already exists: config/sitemap.rb, file not copied" if verbose
       else
         FileUtils.cp(
           SitemapGenerator.templates.template_path(:sitemap_sample),
           File.join(RAILS_ROOT, 'config/sitemap.rb'))
-        puts "created: config/sitemap.rb"
+        puts "created: config/sitemap.rb" if verbose
       end
     end
 
@@ -25,5 +25,30 @@ module SitemapGenerator
     def clean_files
       FileUtils.rm(Dir[File.join(RAILS_ROOT, 'public/sitemap*.xml.gz')])
     end
+
+    # Returns whether this environment is using ActionPack
+    # version 3.0.0 or greater.
+    #
+    # @return [Boolean]
+    def self.rails3?
+      # The ActionPack module is always loaded automatically in Rails >= 3
+      return false unless defined?(ActionPack) && defined?(ActionPack::VERSION)
+
+      version =
+        if defined?(ActionPack::VERSION::MAJOR)
+          ActionPack::VERSION::MAJOR
+        else
+          # Rails 1.2
+          ActionPack::VERSION::Major
+        end
+
+      # 3.0.0.beta1 acts more like ActionPack 2
+      # for purposes of this method
+      # (checking whether block helpers require = or -).
+      # This extra check can be removed when beta2 is out.
+      version >= 3 &&
+        !(defined?(ActionPack::VERSION::TINY) &&
+          ActionPack::VERSION::TINY == "0.beta")
+    end
   end
 end

data/lib/sitemap_generator.rb

@@ -1,19 +1,19 @@
+require 'sitemap_generator/builder'
 require 'sitemap_generator/mapper'
 require 'sitemap_generator/link'
-require 'sitemap_generator/rails_helper'
-require 'sitemap_generator/helper'
 require 'sitemap_generator/link_set'
-require 'sitemap_generator/helper'
 require 'sitemap_generator/templates'
 require 'sitemap_generator/utilities'
-
-require 'sitemap_generator/railtie' if SitemapGenerator::RailsHelper.rails3?
+require 'sitemap_generator/railtie' if SitemapGenerator::Utilities.rails3?
 
 module SitemapGenerator
   silence_warnings do
     VERSION = File.read(File.dirname(__FILE__) + "/../VERSION").strip
-    MAX_ENTRIES = 50_000
-    MAX_IMAGES = 1_000
+    MAX_SITEMAP_FILES    = 50_000        # max sitemap links per index file
+    MAX_SITEMAP_LINKS    = 50_000        # max links per sitemap
+    MAX_SITEMAP_IMAGES   = 1_000         # max images per url
+    MAX_SITEMAP_FILESIZE = 10.megabytes  # bytes
+
     Sitemap = LinkSet.new
   end
 

data/tasks/sitemap_generator_tasks.rake

@@ -1,4 +1,3 @@
-require 'zlib'
 begin
   require 'sitemap_generator'
 rescue LoadError, NameError
@@ -8,7 +7,7 @@ end
 namespace :sitemap do
   desc "Install a default config/sitemap.rb file"
   task :install do
-    SitemapGenerator::Utilities.install_sitemap_rb
+    SitemapGenerator::Utilities.install_sitemap_rb(verbose)
   end
 
   desc "Delete all Sitemap files in public/ directory"
@@ -25,19 +24,8 @@ namespace :sitemap do
   task 'refresh:no_ping' => ['sitemap:create']
 
   task :create => [:environment] do
-    # TODO: Move away from auto-instantiating SitemapGenerator::Sitemap
-    # and move to a more natural Sitemap.new or similar.
-    if SitemapGenerator::RailsHelper.rails3?
-      SitemapGenerator::Sitemap.class_eval do
-        include Rails.application.routes.url_helpers
-      end
-    else
-      require 'action_controller'
-      SitemapGenerator::Sitemap.class_eval do
-        include ActionController::UrlWriter
-      end
-    end
-    SitemapGenerator::Sitemap.create_files
+    SitemapGenerator::Sitemap.verbose = verbose
+    SitemapGenerator::Sitemap.create
   end
 end
 

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
-  - 2
-  - 6
-  version: 0.2.6
+  - 3
+  - 0
+  version: 0.3.0
 platform: ruby
 authors: 
 - Adam Salter
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-05-17 00:00:00 -07:00
+date: 2010-05-21 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -44,11 +44,14 @@ files:
 - Rakefile
 - VERSION
 - lib/sitemap_generator.rb
-- lib/sitemap_generator/helper.rb
+- lib/sitemap_generator/builder.rb
+- lib/sitemap_generator/builder/helper.rb
+- lib/sitemap_generator/builder/sitemap_file.rb
+- lib/sitemap_generator/builder/sitemap_index_file.rb
+- lib/sitemap_generator/interpreter.rb
 - lib/sitemap_generator/link.rb
 - lib/sitemap_generator/link_set.rb
 - lib/sitemap_generator/mapper.rb
-- lib/sitemap_generator/rails_helper.rb
 - lib/sitemap_generator/railtie.rb
 - lib/sitemap_generator/tasks.rb
 - lib/sitemap_generator/templates.rb
@@ -57,8 +60,6 @@ files:
 - rails/uninstall.rb
 - tasks/sitemap_generator_tasks.rake
 - templates/sitemap.rb
-- templates/sitemap_index.builder
-- templates/xml_sitemap.builder
 has_rdoc: true
 homepage: http://github.com/kjvarga/sitemap_generator
 licenses: []

data/lib/sitemap_generator/helper.rb

@@ -1,55 +0,0 @@
-module SitemapGenerator
-  # UrlHelpers are included by the rake tasks.  This is not ideal, but should
-  # suffice until things are better organized.
-  module Helper
-    def self.included(base)
-      base.class_eval do
-        def self.default_url_options(options = nil)
-          { :host => SitemapGenerator::Sitemap.default_host }
-        end
-      end
-    end
-
-    def load_sitemap_rb
-      sitemap_mapper_file = File.join(Rails.root, 'config/sitemap.rb')
-      eval(open(sitemap_mapper_file).read)
-    end
-
-    def url_with_hostname(path)
-      URI.join(SitemapGenerator::Sitemap.default_host, path).to_s
-    end
-
-    def w3c_date(date)
-       date.utc.strftime("%Y-%m-%dT%H:%M:%S+00:00")
-    end
-
-    def ping_search_engines(sitemap_index)
-      require 'open-uri'
-      index_location = CGI.escape(url_with_hostname(sitemap_index))
-      # engines list from http://en.wikipedia.org/wiki/Sitemap_index
-      yahoo_app_id = SitemapGenerator::Sitemap.yahoo_app_id
-      {:google => "http://www.google.com/webmasters/sitemaps/ping?sitemap=#{index_location}",
-        :yahoo => "http://search.yahooapis.com/SiteExplorerService/V1/ping?sitemap=#{index_location}&appid=#{yahoo_app_id}",
-        :ask => "http://submissions.ask.com/ping?sitemap=#{index_location}",
-        :bing => "http://www.bing.com/webmaster/ping.aspx?siteMap=#{index_location}",
-        :sitemap_writer => "http://www.sitemapwriter.com/notify.php?crawler=all&url=#{index_location}"}.each do |engine, link|
-        begin
-          unless SitemapGenerator::Sitemap.yahoo_app_id == false
-            open(link)
-            puts "Successful ping of #{engine.to_s.titleize}" if verbose
-          end
-        rescue Timeout::Error, StandardError => e
-          puts "Ping failed for #{engine.to_s.titleize}: #{e.inspect}" if verbose
-          puts <<-END if engine == :yahoo && verbose
-Yahoo requires an 'AppID' for more than one ping per "timeframe", you can either:
-  - remove yahoo from the ping list (config/sitemap.rb):
-    SitemapGenerator::Sitemap.yahoo_app_id = false
-  - or add your Yahoo AppID to the generator (config/sitemap.rb):
-    SitemapGenerator::Sitemap.yahoo_app_id = "my_app_id"
-For more information: http://developer.yahoo.com/search/siteexplorer/V1/updateNotification.html
-          END
-        end
-      end
-    end
-  end
-end

data/lib/sitemap_generator/rails_helper.rb

@@ -1,28 +0,0 @@
-module SitemapGenerator
-  module RailsHelper
-    # Returns whether this environment is using ActionPack
-    # version 3.0.0 or greater.
-    #
-    # @return [Boolean]
-    def self.rails3?
-      # The ActionPack module is always loaded automatically in Rails >= 3
-      return false unless defined?(ActionPack) && defined?(ActionPack::VERSION)
-
-      version =
-        if defined?(ActionPack::VERSION::MAJOR)
-          ActionPack::VERSION::MAJOR
-        else
-          # Rails 1.2
-          ActionPack::VERSION::Major
-        end
-
-      # 3.0.0.beta1 acts more like ActionPack 2
-      # for purposes of this method
-      # (checking whether block helpers require = or -).
-      # This extra check can be removed when beta2 is out.
-      version >= 3 &&
-        !(defined?(ActionPack::VERSION::TINY) &&
-          ActionPack::VERSION::TINY == "0.beta")
-    end
-  end
-end

data/templates/sitemap_index.builder

@@ -1,23 +0,0 @@
-# encoding: utf-8
-# <?xml version="1.0" encoding="UTF-8"?>
-# <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
-#    <sitemap>
-#       <loc>http://www.example.com/sitemap1.xml.gz</loc>
-#       <lastmod>2004-10-01T18:23:17+00:00</lastmod>
-#    </sitemap>
-#    <sitemap>
-#       <loc>http://www.example.com/sitemap2.xml.gz</loc>
-#       <lastmod>2005-01-01</lastmod>
-#    </sitemap>
-# </sitemapindex>
-
-xml.instruct!
-xml.sitemapindex "xmlns" => "http://www.sitemaps.org/schemas/sitemap/0.9" do
-  sitemap_files.each do |file|
-    xml.sitemap do
-      xml.loc url_with_hostname(File.basename(file))
-      xml.lastmod w3c_date(File.mtime(file))
-    end
-  end
-end
-

data/templates/xml_sitemap.builder

@@ -1,38 +0,0 @@
-# encoding: utf-8
-xml.instruct!
-xml.urlset "xmlns:xsi" => "http://www.w3.org/2001/XMLSchema-instance",
-  "xsi:schemaLocation" => "http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/siteindex.xsd",
-  "xmlns:image" => "http://www.google.com/schemas/sitemap-image/1.1",
-  "xmlns" => "http://www.sitemaps.org/schemas/sitemap/0.9" do
-
-    links.each_with_index do |link,index|
-      buffer_url = ""
-      url = Builder::XmlMarkup.new(:target=>buffer_url)
-      url.url do
-        url.loc        link[:loc]
-        url.lastmod    w3c_date(link[:lastmod])   if link[:lastmod]
-        url.changefreq link[:changefreq]          if link[:changefreq]
-        url.priority   link[:priority]            if link[:priority]
-
-        unless link[:images].blank?
-          link[:images].each do |image|
-            url.image:image do
-              url.image :loc, image[:loc]
-              url.image :caption, image[:caption]             if image[:caption]
-              url.image :geo_location, image[:geo_location]   if image[:geo_location]
-              url.image :title, image[:title]                 if image[:title]
-              url.image :license, image[:license]             if image[:license]
-            end
-          end
-        end
-      end
-
-      if (buffer+buffer_url).size < 10.megabytes
-        xml << buffer_url
-      else
-        slice_index = index
-        break
-      end
-    end
-end
-