sitemap_generator 0.1.0

data/.autotest

@@ -0,0 +1,36 @@
+class Autotest
+  ##
+  # Convert a path in a string, s, into a class name, changing
+  # underscores to CamelCase, etc.
+ 
+  def path_to_classname(s)
+    sep = File::SEPARATOR
+    f = s.sub(/^test#{sep}/, '').sub(/\.rb$/, '').split(sep)
+    f = f.map { |path| path.split(/_|(\d+)/).map { |seg| seg.capitalize }.join }
+    f = f.map { |path| path =~ /Test$/ ? path : "#{path}Test"  }
+    f.join('::')
+  end
+end
+
+Autotest.add_hook :initialize do |at|
+  unless ARGV.empty?
+    if ARGV[0] == '-d'
+      at.find_directories = ARGV[1..-1].dup
+    else
+      at.find_directories = []
+      at.extra_files = ARGV.dup
+    end
+  end
+  
+  # doesn't seem to work
+  # at.clear_mappings
+  
+  at.add_mapping(/^lib\/.*\.rb$/) do |filename, _|
+    possible = File.basename(filename, 'rb').gsub '_', '_?'
+    files_matching %r%^test/.*#{possible}_test\.rb$%
+  end
+
+  at.add_mapping(/^test.*\/.*test\.rb$/) do |filename, _|
+    filename
+  end
+end

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 [name of plugin creator]
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,131 @@
+SitemapGenerator
+================
+
+This plugin enables ['enterprise-class'][enterprise_class] Google Sitemaps to be easily generated for a Rails site as a rake task, using a simple 'Rails Routes'-like DSL.
+
+Raison d'être
+-------
+
+Most of the Sitemap plugins out there seem to try to recreate the Sitemap links by iterating the Rails routes. In some cases this is possible, but for a great deal of cases it isn't. 
+
+a) There are probably quite a few routes in your routes file that don't need inclusion in the Sitemap. (AJAX routes I'm looking at you.)
+
+and
+
+b) How would you infer the correct series of links for the following route?
+
+    map.zipcode 'location/:state/:city/:zipcode', :controller => 'zipcode', :action => 'index'
+    
+Don't tell me it's trivial, because it isn't. It just looks trivial.
+
+So my idea is to have another file similar to 'routes.rb' called 'sitemap.rb', where you can define what goes into the Sitemap.
+
+Here's my solution:
+
+    Zipcode.find(:all, :include => :city).each do |z|
+      sitemap.add zipcode_path(:state => z.city.state, :city => z.city, :zipcode => z)
+    end
+
+Easy hey?
+
+Other Sitemap settings for the link, like `lastmod`, `priority`, `changefreq` and `host` are entered automatically, although you can override them if you need to.
+
+Other "difficult" Sitemap issues, solved by this plugin:
+
+- Support for more than 50,000 urls (using a Sitemap Index file)
+- Gzip of Sitemap files
+- Variable priority of links
+- Paging/sorting links (e.g. my_list?page=3)
+- SSL host links (e.g. https:)
+- Rails apps which are installed on a sub-path (e.g. example.com/blog_app/)
+
+Installation
+=======
+
+1. Install plugin as normal
+
+    <code>./script/plugin install git://github.com/adamsalter/sitemap_generator-plugin.git</code>
+
+2. Installation should create a 'config/sitemap.rb' file which will contain your logic for generation of the Sitemap files. (If you want to recreate this file manually run `rake sitemap:install`)
+
+3. Run `rake sitemap:refresh` as needed to create Sitemap files. This will also ping all the ['major'][sitemap_engines] search engines. (if you want to disable all non-essential output run the rake task thusly `rake -s sitemap:refresh SILENT=true`)
+
+    Sitemaps with many urls (100,000+) take quite a long time to generate, so if you need to refresh your Sitemaps regularly you can set the rake task up as a cron job. Most cron agents will only send you an email if there is output from the cron task.
+
+4. Finally, and optionally, add the following to your robots.txt file.
+
+    <code>Sitemap: &lt;hostname>/sitemap_index.xml.gz</code>
+    
+    The robots.txt Sitemap URL should be the complete URL to the Sitemap Index, such as: `http://www.example.org/sitemap_index.xml.gz`
+
+Example 'config/sitemap.rb'
+==========
+
+    # Set the host name for URL creation
+    SitemapGenerator::Sitemap.default_host = "http://www.example.com"
+
+    SitemapGenerator::Sitemap.add_links do |sitemap|
+      # Put links creation logic here.
+      #
+      # The Root Path ('/') and Sitemap Index file are added automatically.
+      # Links are added to the Sitemap output in the order they are specified.
+      #
+      # Usage: sitemap.add path, options
+      #        (default options are used if you don't specify them)
+      #
+      # Defaults: :priority => 0.5, :changefreq => 'weekly', 
+      #           :lastmod => Time.now, :host => default_host
+
+  
+      # Examples:
+  
+      # add '/articles'
+      sitemap.add articles_path, :priority => 0.7, :changefreq => 'daily'
+
+      # add all individual articles
+      Article.find(:all).each do |a|
+        sitemap.add article_path(a), :lastmod => a.updated_at
+      end
+
+      # add merchant path
+      sitemap.add '/purchase', :priority => 0.7, :host => "https://www.example.com"
+  
+    end
+
+Notes
+=======
+
+1) Tested/working on Rails 1.x.x <=> 2.x.x, no guarantees made for Rails 3.0.
+
+2) For large sitemaps it may be useful to split your generation into batches to avoid running out of memory. E.g.:
+
+    # add movies
+    Movie.find_in_batches(:batch_size => 1000) do |movies|
+      movies.each do |movie|
+        sitemap.add "/movies/show/#{movie.to_param}", :lastmod => movie.updated_at, :changefreq => 'weekly'
+      end
+    end
+
+
+Known Bugs
+========
+
+- Sitemaps.org [states][sitemaps_org] that no Sitemap XML file should be more than 10Mb uncompressed. The plugin will warn you about this, but does nothing to avoid it (like move some URLs into a later file).
+- There's no check on the size of a URL which [isn't supposed to exceed 2,048 bytes][sitemaps_xml].
+- Currently only supports one Sitemap Index file, which can contain 50,000 Sitemap files which can each contain 50,000 urls, so it _only_ supports up to 2,500,000,000 (2.5 billion) urls. I personally have no need of support for more urls, but plugin could be improved to support this.
+
+Follow me on:
+---------
+
+>  Twitter: [twitter.com/adamsalter](http://twitter.com/adamsalter)  
+>  Github: [github.com/adamsalter](http://github.com/adamsalter)
+
+Copyright (c) 2009 Adam @ [Codebright.net][cb], released under the MIT license
+
+[enterprise_class]:https://twitter.com/dhh/status/1631034662 "I use enterprise in the same sense the Phusion guys do - i.e. Enterprise Ruby. Please don't look down on my use of the word 'enterprise' to represent being a cut above. It doesn't mean you ever have to work for a company the size of IBM. Or constantly fight inertia, writing crappy software, adhering to change management practices and spending hours in meetings... Not that there's anything wrong with that - Wait, what?"
+[sitemap_engines]:http://en.wikipedia.org/wiki/Sitemap_index "http://en.wikipedia.org/wiki/Sitemap_index"
+[sitemaps_org]:http://www.sitemaps.org/protocol.php "http://www.sitemaps.org/protocol.php"
+[sitemaps_xml]:http://www.sitemaps.org/protocol.php#xmlTagDefinitions "XML Tag Definitions"
+[sitemap_generator_usage]:http://wiki.github.com/adamsalter/sitemap_generator-plugin/sitemapgenerator-usage "http://wiki.github.com/adamsalter/sitemap_generator-plugin/sitemapgenerator-usage"
+[boost_juice]:http://www.boostjuice.com.au/ "Mmmm, sweet, sweet Boost Juice."
+[cb]:http://codebright.net "http://codebright.net"

data/Rakefile

@@ -0,0 +1,29 @@
+require 'rake/testtask'
+require 'find'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "sitemap_generator"
+    gem.summary = %Q{This plugin enables 'enterprise-class' Google Sitemaps to be easily generated for a Rails site as a rake task}
+    gem.description = %Q{This plugin enables 'enterprise-class' Google Sitemaps to be easily generated for a Rails site as a rake task}
+    gem.email = "adam@salter.net "
+    gem.homepage = "http://github.com/adamsalter/sitemap_generator-plugin"
+    gem.authors = ["Adam Salter"]
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+desc 'Default: run unit tests.'
+task :default => :test
+
+desc 'Test ActiveScaffold.'
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = true
+end
+

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/init.rb

@@ -0,0 +1,2 @@
+# Include hook code here
+

data/install.rb

@@ -0,0 +1,13 @@
+# Install hook code here
+
+# Copy sitemap_template.rb to config/sitemap.rb
+require 'fileutils'
+current_dir = File.dirname(__FILE__)
+sitemap_template = File.join(current_dir, 'templates/sitemap.rb')
+new_sitemap = File.join(RAILS_ROOT, 'config/sitemap.rb')
+if File.exist?(new_sitemap)
+  puts "already exists: config/sitemap.rb, file not copied"
+else
+  puts "created: config/sitemap.rb"
+  FileUtils.cp(sitemap_template, new_sitemap)
+end

data/lib/sitemap_generator.rb

@@ -0,0 +1,10 @@
+require 'sitemap_generator/mapper'
+require 'sitemap_generator/link'
+require 'sitemap_generator/link_set'
+require 'sitemap_generator/helper'
+require 'sitemap_generator/tasks'
+
+module SitemapGenerator
+  Sitemap = LinkSet.new
+end
+  

data/lib/sitemap_generator/helper.rb

@@ -0,0 +1,59 @@
+require 'action_controller'
+require 'action_controller/test_process'
+begin
+  require 'application_controller'
+rescue LoadError
+  # Rails < 2.3
+  require 'application'
+end
+
+module SitemapGenerator
+  module Helper
+    def load_sitemap_rb
+      controller = ApplicationController.new
+      controller.request = ActionController::TestRequest.new
+      controller.params = {}
+      controller.send(:initialize_current_url)  
+      b = controller.instance_eval{binding}
+      sitemap_mapper_file = File.join(RAILS_ROOT, 'config/sitemap.rb')
+      eval(open(sitemap_mapper_file).read, b)
+    end
+    
+    def url_with_hostname(path)
+      URI.join(Sitemap.default_host, path).to_s
+    end
+    
+    def w3c_date(date)
+       date.utc.strftime("%Y-%m-%dT%H:%M:%S+00:00")
+    end
+    
+    def ping_search_engines(sitemap_index)
+      require 'open-uri'
+      index_location = CGI.escape(url_with_hostname(sitemap_index))
+      # engines list from http://en.wikipedia.org/wiki/Sitemap_index
+      yahoo_app_id = SitemapGenerator::Sitemap.yahoo_app_id
+      {:google => "http://www.google.com/webmasters/sitemaps/ping?sitemap=#{index_location}",
+        :yahoo => "http://search.yahooapis.com/SiteExplorerService/V1/ping?sitemap=#{index_location}&appid=#{yahoo_app_id}",
+        :ask => "http://submissions.ask.com/ping?sitemap=#{index_location}",
+        :bing => "http://www.bing.com/webmaster/ping.aspx?siteMap=#{index_location}",
+        :sitemap_writer => "http://www.sitemapwriter.com/notify.php?crawler=all&url=#{index_location}"}.each do |engine, link|
+        begin
+          unless SitemapGenerator::Sitemap.yahoo_app_id == false
+            open(link)
+            puts "Successful ping of #{engine.to_s.titleize}" unless ENV['SILENT'].present?
+          end
+        rescue StandardError => e
+          puts "Ping failed for #{engine.to_s.titleize}: #{e.inspect}"
+          puts <<-END if engine == :yahoo
+Yahoo requires an 'AppID' for more than one ping per "timeframe", you can either:
+  - remove yahoo from the ping list (config/sitemap.rb):
+    SitemapGenerator::Sitemap.yahoo_app_id = false
+  - or add your Yahoo AppID to the generator (config/sitemap.rb):
+    SitemapGenerator::Sitemap.yahoo_app_id = "my_app_id"
+For more information: http://developer.yahoo.com/search/siteexplorer/V1/updateNotification.html
+          END
+        end
+      end
+    end
+  end
+end

data/lib/sitemap_generator/link.rb

@@ -0,0 +1,19 @@
+
+module SitemapGenerator
+  class Link
+    class << self
+      def generate(path, options = {})
+        options.assert_valid_keys(:priority, :changefreq, :lastmod, :host)
+        options.reverse_merge!(:priority => 0.5, :changefreq => 'weekly', :lastmod => Time.now, :host => Sitemap.default_host)
+        {
+          :path => path,
+          :priority => options[:priority],
+          :changefreq => options[:changefreq],
+          :lastmod => options[:lastmod],
+          :host => options[:host],
+          :loc => URI.join(options[:host], path).to_s
+        }
+      end
+    end
+  end
+end

data/lib/sitemap_generator/link_set.rb

@@ -0,0 +1,28 @@
+module SitemapGenerator
+  class LinkSet
+    attr_accessor :default_host, :yahoo_app_id, :links
+    
+    def initialize
+      @links = []
+    end
+
+    def default_host=(host)
+      @default_host = host
+      add_default_links
+    end
+
+    def add_default_links
+      # Add default links
+      @links << Link.generate('/', :lastmod => Time.now, :changefreq => 'always', :priority => 1.0)
+      @links << Link.generate('/sitemap_index.xml.gz', :lastmod => Time.now, :changefreq => 'always', :priority => 1.0)
+    end
+    
+    def add_links
+      yield Mapper.new(self)
+    end
+    
+    def add_link(link)
+      @links << link
+    end
+  end
+end

data/lib/sitemap_generator/mapper.rb

@@ -0,0 +1,16 @@
+
+module SitemapGenerator
+  # Generator instances are used to build links.
+  # The object passed to the add_links block in config/sitemap.rb is a Generator instance.
+  class Mapper
+    attr_accessor :set
+    
+    def initialize(set)
+      @set = set
+    end
+    
+    def add(loc, options = {})
+      set.add_link Link.generate(loc, options)
+    end
+  end
+end

data/lib/sitemap_generator/tasks.rb

@@ -0,0 +1 @@
+load "#{File.dirname(__FILE__)}/../../tasks/sitemap_generator_tasks.rake"

data/tasks/sitemap_generator_tasks.rake

@@ -0,0 +1,74 @@
+require 'zlib'
+
+namespace :sitemap do
+
+  desc "Install a default config/sitemap.rb file"
+  task :install do
+    load File.expand_path(File.join(File.dirname(__FILE__), "..", "install.rb"))
+  end
+
+  desc "Delete all Sitemap files in public/ directory"
+  task :clean do
+    sitemap_files = Dir[File.join(RAILS_ROOT, 'public/sitemap*.xml.gz')]
+    FileUtils.rm sitemap_files
+  end
+
+  desc "Create Sitemap XML files in public/ directory"
+  desc "Create Sitemap XML files in public/ directory (set SILENT=true for no output)"
+  task :refresh => ['sitemap:create'] do
+    ping_search_engines("sitemap_index.xml.gz")
+  end
+
+  desc "Create Sitemap XML files (don't ping search engines)"
+  task 'refresh:no_ping' => ['sitemap:create'] do
+  end
+
+  task :create => [:environment] do
+    include SitemapGenerator::Helper
+    include ActionView::Helpers::NumberHelper
+
+    start_time = Time.now
+
+    # update links from config/sitemap.rb
+    load_sitemap_rb
+
+    raise(ArgumentError, "Default hostname not defined") if SitemapGenerator::Sitemap.default_host.blank?
+
+    links_grps = SitemapGenerator::Sitemap.links.in_groups_of(50000, false)
+    raise(ArgumentError, "TOO MANY LINKS!! I really thought 2,500,000,000 links would be enough for anybody!") if links_grps.length > 50000
+
+    Rake::Task['sitemap:clean'].invoke
+
+    # render individual sitemaps
+    sitemap_files = []
+    xml_sitemap_template = File.join(File.dirname(__FILE__), '../templates/xml_sitemap.builder')
+    links_grps.each_with_index do |links, index|
+      buffer = ''
+      xml = Builder::XmlMarkup.new(:target=>buffer)
+      eval(open(xml_sitemap_template).read, binding)
+      filename = File.join(RAILS_ROOT, "public/sitemap#{index+1}.xml.gz")
+      Zlib::GzipWriter.open(filename) do |gz|
+        gz.write buffer
+      end
+      puts "+ #{filename}" unless ENV['SILENT'].present?
+      puts "** Sitemap too big! The uncompressed size exceeds 10Mb" if (buffer.size > 10 * 1024 * 1024) && ENV['SILENT'].blank?
+      sitemap_files << filename
+    end
+
+    # render index
+    sitemap_index_template = File.join(File.dirname(__FILE__), '../templates/sitemap_index.builder')
+    buffer = ''
+    xml = Builder::XmlMarkup.new(:target=>buffer)
+    eval(open(sitemap_index_template).read, binding)
+    filename = File.join(RAILS_ROOT, "public/sitemap_index.xml.gz")
+    Zlib::GzipWriter.open(filename) do |gz|
+      gz.write buffer
+    end
+    puts "+ #{filename}" unless ENV['SILENT'].present?
+    puts "** Sitemap Index too big! The uncompressed size exceeds 10Mb" if (buffer.size > 10 * 1024 * 1024) && ENV['SILENT'].blank?
+
+    stop_time = Time.now
+    puts "Sitemap stats: #{number_with_delimiter(SitemapGenerator::Sitemap.links.length)} links, " + ("%dm%02ds" % (stop_time - start_time).divmod(60)) unless ENV['SILENT'].present?
+
+  end
+end