ar_sitemapper 1.0.0

data/lib/sitemapper/engine.rb

@@ -0,0 +1,26 @@
+require "rails"
+require 'sitemapper/loader'
+
+module AegisNet
+  module Sitemapper
+
+    class Engine < Rails::Engine
+
+      initializer 'ar_sitemapper.load_app_root' do |app|
+        AegisNet::Sitemapper.sitemap_file ||= File.join(app.root, "config", "sitemaps.yml")
+      end
+
+      initializer 'ar_sitemapper.hook_into_active_record' do
+        ActiveSupport.on_load(:active_record) do
+          ::ActiveRecord::Base.send :include, AegisNet::Sitemapper::ActiveRecord::Builder
+        end
+      end
+
+      rake_tasks do
+        load 'tasks/sitemapper.rake'
+      end
+
+    end
+
+  end
+end

data/lib/sitemapper/generator.rb

@@ -0,0 +1,100 @@
+module AegisNet
+  module Sitemapper
+    class Generator
+
+      require "zlib"
+
+      VALID_GENERATOR_OPTIONS = [:file, :filename, :gzip, :xmlns]
+
+      # Generates an XML Sitemap file with +entries+. The output is written
+      # to a file if a filename is given or to stdout otherwise. Expects a
+      # block.
+      #
+      # === Parameters
+      # * +entries+: Enumerable to iterate through
+      # * +options+: an optional Hash. See below for supported options
+      #
+      # === Available Options
+      # * +:file+: full path to output file. If +:filename+ ends with .gz, Gzip-compression is activated
+      # * +:gzip+: force GZip compression if set to +true+
+      # * +:xmlns+: XML namespace to use, defaults to http://www.sitemaps.org/schemas/sitemap/0.9
+      #
+      # === Example
+      #  sites = [
+      #    { :url => "http://example.com/your/static/content1.html", :freq => "always",  :prio => "1.0" },
+      #    { :url => "http://example.com/your/static/content2.html", :freq => "monthly", :prio => "0.3" },
+      #  ]
+      #
+      #  AegisNet::Sitemapper::Generator.create(sites) do |site, xml|
+      #    xml.loc site[:url]
+      #    xml.changefreq site[:freq]
+      #    xml.priority site[:prio]
+      #  end
+      #
+      def self.create entries, options = {}, &block
+        if block_given?
+          options.symbolize_keys!
+          options.assert_valid_keys(VALID_GENERATOR_OPTIONS)
+          xmlns    = options[:xmlns] || "http://www.sitemaps.org/schemas/sitemap/0.9"
+          gzip     = options[:gzip]  ||  /\.gz$/.match(options[:file])
+          filename = options[:file] ? options[:file].gsub(/\.gz$/, '') : nil
+
+          if entries.size > 50_000
+            part_number = 0
+            entries.each_slice(50_000) do |part|
+              part_number = part_number.next
+              part_fn = filename.gsub('.xml', ".#{part_number}.xml")
+
+              create_one_sitemap(part, xmlns, part_fn, gzip, &block)
+            end
+          else
+            create_one_sitemap(entries, xmlns, filename, gzip, &block)
+          end
+        end
+      end
+
+      # Infer full local path and sitemap filename by class name. Adds .xml.gz
+      #
+      # === Parameters
+      # * +klass+: class name
+      def self.default_filename(klass)
+        config = AegisNet::Sitemapper::Loader.load_config
+        File.join(config[:local_path], "sitemap_#{klass.to_s.underscore.pluralize}.xml.gz") if config[:local_path]
+      end
+
+      def self.create_necessary_directories(filename)
+        FileUtils.mkpath( File.dirname(filename) )
+      end
+
+      def self.create_one_sitemap(entries, xmlns, filename, gzip, &block)
+        write_one_sitemap(
+          generate_one_sitemap(entries, xmlns, &block),
+          filename,
+          gzip
+        )
+      end
+
+      def self.generate_one_sitemap(entries, xmlns, &block)
+        xml = Builder::XmlMarkup.new(:indent => 2)
+        xml.instruct!
+        xml.urlset "xmlns" => xmlns do
+          entries.each do |entry|
+            xml.url { block.call(entry, xml) } rescue nil # TODO handle me / pass upwards
+          end
+        end
+        xml
+      end
+
+      def self.write_one_sitemap(xml, filename, gzip)
+        # Either write to file or to stdout
+        if filename
+          create_necessary_directories(filename)
+          File.open(filename, "w") { |file| file.puts xml.target! }
+          Zlib::GzipWriter.open("#{filename}.gz") {|gz| gz.write xml.target! } if gzip
+        else
+          $stdout.puts xml.target!
+        end
+      end
+    end
+  end
+end

data/lib/sitemapper/index.rb

@@ -0,0 +1,84 @@
+module AegisNet # :nodoc:
+  module Sitemapper # :nodoc:
+    # :doc:
+
+    class Index
+
+      attr_reader :host, :sitemaps
+
+      def initialize(options = {})
+        options.symbolize_keys!
+        @sitemaps = []
+
+        @config = AegisNet::Sitemapper::Loader.load_config
+        @host   = @config[:default_host]
+        @file   = File.join("#{@config[:local_path]}", @config[:index]["sitemapfile"])
+
+        @static   = @config[:static]
+        @models   = @config[:models]
+        @includes = @config[:index]["includes"]
+
+        # Validate all variables
+        raise(ArgumentError, "No filename specified") if @file.nil?
+
+        # Static Sitemap
+        if @static
+          sitemap_options = {:loc => @static["sitemapfile"], :lastmod => @static["lastmod"]}
+          @sitemaps << AegisNet::Sitemapper::Urlset.new(sitemap_options)
+        end
+
+        # Include additional sitemaps
+        @includes.each do |sitemap|
+          sitemap_options = {:loc => sitemap["loc"] }
+          sitemap_options.merge!(:lastmod => sitemap["lastmod"]) if sitemap["lastmod"]
+          @sitemaps << AegisNet::Sitemapper::Sitemap.new(sitemap_options)
+        end
+
+        @models.each do |sitemap|
+          klass = sitemap.first.camelize.constantize
+          count = klass.count
+
+          order_opts = {}
+          order_opts = { :order => :created_at } if klass.column_names.include?("created_at")
+          lastmod = sitemap.last["lastmod"] || klass.last(order_opts).created_at
+
+          if count <= 50_000
+            sitemap_options = {:loc => sitemap.last["sitemapfile"], :lastmod => lastmod}
+            @sitemaps << AegisNet::Sitemapper::Urlset.new(sitemap_options)
+          else
+            1.upto( (count / 50_000.0).ceil ) do |part_number|
+              sitemap_options = {
+                :loc => sitemap.last["sitemapfile"].gsub("xml", "#{part_number}.xml"),
+                :lastmod => lastmod
+              }
+              @sitemaps << AegisNet::Sitemapper::Urlset.new(sitemap_options)
+            end
+          end
+        end
+      end
+
+      def self.create!(options = {})
+        index = self.new(options)
+        index.create!
+      end
+
+      def create!
+        xml = Builder::XmlMarkup.new(:indent => 2)
+        xml.instruct!
+
+        xml.sitemapindex "xmlns" => "http://www.sitemaps.org/schemas/sitemap/0.9" do
+
+          @sitemaps.each do |sitemap|
+            location = sitemap.loc.gsub(/^\//, '')
+            xml.sitemap do
+              xml.loc         "http://#{@host}/#{location}"
+              xml.lastmod     sitemap.lastmod.to_date if sitemap.lastmod rescue nil # TODO handle properly
+            end
+          end
+        end
+        File.open(@file, "w") { |file| file.puts xml.target! }
+      end
+    end
+
+  end
+end

data/lib/sitemapper/loader.rb

@@ -0,0 +1,39 @@
+require 'sitemapper/generator'
+require 'sitemapper/sitemap'
+require 'sitemapper/urlset'
+require 'sitemapper/index'
+require 'sitemapper/pinger'
+require 'sitemapper/active_record/builder'
+
+module AegisNet
+  module Sitemapper
+
+    class Loader
+      # Loads the sitemap configuration from Rails.root/config/sitemap.yml
+      def self.load_config
+        # TODO verify file integrity
+        erb = ERB.new(File.read(AegisNet::Sitemapper.sitemap_file))
+        AegisNet::Sitemapper.configuration ||= HashWithIndifferentAccess.new(YAML.load(StringIO.new(erb.result)))
+      end
+
+      # Interprets +string+ as Ruby code representing a Proc and exectutes it.
+      #
+      # === Parameters
+      # * +string+: Ruby (Proc) code to be executed
+      # All other arguments will be passed to the Proc
+      #
+      # === Examples
+      #  AegisNet::Sitemapper::Loader.proc_loader('Proc.new{"foo"}')
+      #  => "foo"
+      #
+      #  proc_str = 'Proc.new{|n| n}'
+      #  AegisNet::Sitemapper::Loader.proc_loader(proc_str, "hello world")
+      #  => "hello world"
+      def self.proc_loader(string, *args)
+        # TODO lambdas
+        eval(string).call(*args)
+      end
+
+    end
+  end
+end

data/lib/sitemapper/pinger.rb

@@ -0,0 +1,21 @@
+module AegisNet # :nodoc:
+  module Sitemapper # :nodoc:
+    # :doc:
+
+    class Pinger
+
+      require "net/http"
+
+      def self.ping!
+        config = AegisNet::Sitemapper::Loader.load_config
+        if config[:ping] and config[:default_host] and config[:index]
+          config[:pings].each do |ping_url|
+            url = ping_url + config[:default_host] + "/" + config[:index]["sitemapfile"]
+            Net::HTTP.get_response(URI.parse(url)) rescue nil
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/sitemapper/sitemap.rb

@@ -0,0 +1,22 @@
+module AegisNet # :nodoc:
+  module Sitemapper # :nodoc:
+    # :doc:
+    class Sitemap
+      attr_reader :lastmod, :loc
+
+      def initialize(options = {})
+        options.symbolize_keys!
+        options.assert_valid_keys(:changefreq, :lastmod, :loc, :priority)
+        @changefreq = options[:changefreq] || "weekly"
+        @lastmod    = options[:lastmod]
+        @loc        = options[:loc]
+        @priority   = options[:priority] || 0.5
+      end
+
+      def changefreq(freq = nil); freq ? @changefreq = freq : @changefreq;  end
+      def lastmod(lastmod = nil); lastmod ? @lastmod = lastmod : @lastmod; end
+      def loc(loc = nil); loc ? @loc = loc : @loc; end
+      def priority(prio = nil); prio ? @priority = prio : @priority; end
+    end
+  end
+end

data/lib/sitemapper/urlset.rb

@@ -0,0 +1,76 @@
+module AegisNet # :nodoc:
+  module Sitemapper # :nodoc:
+    # :doc:
+
+   class Urlset < AegisNet::Sitemapper::Sitemap
+
+      def create!
+        xml = Builder::XmlMarkup.new(:indent => 2)
+        xml.instruct!
+
+        xml.urlset "xmlns" => "http://www.sitemaps.org/schemas/sitemap/0.9" do
+
+          @sitemaps.each do |sitemap|
+            location = sitemap.loc.gsub(/^\//, '')
+            xml.url do
+              xml.loc         "http://#{@host}/#{location}"
+              xml.lastmod     sitemap.lastmod if sitemap.lastmod
+              xml.changefreq  sitemap.changefreq
+              xml.priority    sitemap.priority
+            end
+          end
+
+        end
+        File.open(@file, "w") { |file| file.puts xml.target! }
+      end
+
+      # Short-hand for Urlset#new and Urlset#create!
+      def self.create!(options = {})
+        sitemap = self.new(options)
+        sitemap.create!
+      end
+
+      # Generate Urlset sitemaps listed in sitemaps.yml
+      def self.build_all!
+        config = AegisNet::Sitemapper::Loader.load_config
+
+        # Generate sitemaps for AR Models dynamically by yml instructions
+        if config[:models]
+          config[:models].each do |ar_map|
+            opts  = ar_map.last
+            klass = ar_map.first.camelize.constantize
+
+            build_opts = { :file => File.join("#{config[:local_path]}", opts["sitemapfile"]) }
+            build_opts.merge!( :conditions => opts["conditions"])  if opts["conditions"]
+
+            scope = opts["scope"].present? ? "#{opts["scope"]}" : :all
+
+            klass.build_sitemap scope, build_opts do |object, xml|
+              if opts["loc"].starts_with?("Proc")
+                xml.loc AegisNet::Sitemapper::Loader.proc_loader(opts["loc"], object)
+              else
+                xml.loc opts["loc"]
+              end
+              xml.lastmod    object.updated_at.to_date
+              xml.changefreq opts["changefreq"] || "weekly"
+              xml.priority   opts["priority"] || 0.5
+            end
+          end
+        end
+
+        # Find misc. sitemap data and generate a single static one
+        if config[:static]
+          entries = config[:static]["urlset"]
+          file    = File.join("#{config[:local_path]}", config[:static]["sitemapfile"])
+          AegisNet::Sitemapper::Generator.create(entries, :file => file) do |entry, xml|
+            xml.loc        entry["loc"]
+            xml.lastmod    entry["lastmod"] if entry["lastmod"]
+            xml.changefreq entry["changefreq"] if entry["changefreq"]
+            xml.priority   entry["priority"] || 0.5
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/sitemapper/version.rb

@@ -0,0 +1,5 @@
+module AegisNet
+  module Sitemapper
+    VERSION = "1.0.0"
+  end
+end

data/lib/tasks/sitemapper.rake

@@ -0,0 +1,26 @@
+# encoding: utf-8
+namespace :sitemapper do
+
+  desc "Rebuilds all sitemaps"
+  task :rebuild => [:environment, :build_from_config]
+
+  desc "Notifies search-engines about the index-sitemap"
+  task :ping => :environment do
+    ActiveRecord::Migration.say_with_time "Pinging searchengines" do
+      AegisNet::Sitemapper::Pinger.ping!
+    end
+  end
+
+  desc "Rebuild everything from config/sitemapper.yml"
+  task :build_from_config => :environment do
+    include Rails.application.routes.url_helpers
+    config = AegisNet::Sitemapper::Loader.load_config
+    default_url_options[:host] = config["default_host"]
+
+    ActiveRecord::Migration.say_with_time "Rebuilding sitemaps from #{AegisNet::Sitemapper.sitemap_file}" do
+      AegisNet::Sitemapper::Urlset.build_all!
+      AegisNet::Sitemapper::Index.create!
+    end
+  end
+
+end

data/templates/sitemaps.yml

@@ -0,0 +1,34 @@
+default_host: "www.example.com"
+local_path: <%= File.join Rails.root, "public", "sitemaps" %>
+ping: true
+
+index:
+  sitemapfile: "sitemap_index.xml"
+  includes:
+    -
+      loc: foo_sitemap.xml
+static:
+  sitemapfile: "sitemapfoobar.xml.gz"
+  urlset:
+    -
+      loc: "http://www.example.com/static/page"
+      changefreq: weekly
+      priority: 0.1
+    -
+      loc: "http://www.example.com/important/static/page"
+      changefreq: daily
+      priority: 1.0
+
+models:
+  foo_bar:
+    conditions: "foo = bar"
+    sitemapfile: sitemap_foo_bars.xml.gz
+    loc: Proc.new {|o| foo_bar_url(o) }
+    changefreq: weekly
+    priority: 0.7
+
+pings:
+  - http://submissions.ask.com/ping?sitemap=
+  - http://www.google.com/webmasters/sitemaps/ping?sitemap=
+  - http://search.yahooapis.com/SiteExplorerService/V1/updateNotification?appid=YahooDemo&url=
+  - http://www.bing.com/webmaster/ping.aspx?siteMap