bcms_sitemap 0.9.0

data/config/initializers/bcms_sitemap.rb

@@ -0,0 +1,4 @@
+# State what models to publish as a hash. 
+# The keys are the plural names of the models.
+# The values should be the scope to be used, formed as string
+Cms::SitemapSubmitter.publish_models = {:pages => 'published.not_hidden', :news_articles => 'released'}

data/config/initializers/init_module.rb

@@ -0,0 +1 @@
+require File.join(File.dirname(__FILE__), "..", "..", "lib", "bcms_sitemap")

data/db/migrate/20100212083005_create_search_engines.rb

@@ -0,0 +1,24 @@
+class CreateSearchEngines < ActiveRecord::Migration
+  def self.up
+    create_table :search_engines do |t|
+      t.string :name
+      t.boolean :enabled
+      t.string :url
+      t.string :verification_file
+      t.text :verification_content
+      t.integer :last_status
+      t.timestamp :submitted_at
+      t.timestamps
+    end
+    # Create a sample search engine
+    SearchEngine.create :name => 'Google', :url => 'http://www.google.com/webmasters/tools/ping?sitemap='
+    SearchEngine.create :name => 'Yahoo', :url => 'http://search.yahooapis.com/SiteExplorerService/V1/ping?sitemap='
+    SearchEngine.create :name => 'Ask', :url => 'http://submissions.ask.com/ping?sitemap='
+    SearchEngine.create :name => 'Live/Bing', :url => 'http://www.bing.com/webmaster/ping.aspx?siteMap='
+    SearchEngine.create :name => 'Moreover', :url => 'http://api.moreover.com/ping?u='
+  end
+
+  def self.down
+    drop_table :search_engines
+  end
+end

data/lib/bcms_sitemap.rb

@@ -0,0 +1,3 @@
+require 'bcms_sitemap/routes'
+require 'bcms_sitemap/sitemap_submitter'
+require File.join(File.dirname(__FILE__), '..', 'app', 'controllers', 'sitemaps_controller')

data/lib/bcms_sitemap/routes.rb

@@ -0,0 +1,10 @@
+module Cms::Routes
+  def routes_for_bcms_sitemap
+    sitemaps 'sitemaps.xml', :controller => :sitemaps, :action => 'index'
+    sitemap_for_model 'sitemaps/:model.xml', :controller => :sitemaps, :action => 'model', 
+      :requirements => { :model => /#{Cms::SitemapSubmitter.models.join('|')}/ }
+    namespace :cms do |cms|
+      cms.resources :search_engines
+    end
+  end
+end

data/lib/bcms_sitemap/sitemap_submitter.rb

@@ -0,0 +1,110 @@
+require 'cgi'
+class Cms::SitemapSubmitter
+  include ActionController::UrlWriter
+  include ActionController::Caching::Pages
+  
+  class << self
+    
+    # Checks to see if there has been any updates since last time.
+    # If so, clears the cache for the relevant model and submits the url to the search engine's that have not yet been notified
+    def run
+      submit_time = SearchEngine.enabled.minimum(:submitted_at)
+      
+      # collect timestamp for all models. We don't want to expire more pages than necessary
+      timestamps = {}
+      @models.each do |model|
+        last_update = model.classify.constantize.bcms_sitemap_last_update
+        timestamps[model] = last_update if last_update
+      end
+      last_update = timestamps.values.compact.max
+      # try this {}.values.compact.max
+      if last_update && (submit_time.nil? || submit_time < last_update)
+        # This is a lazy cleaning of cache
+        expire_page :controller => 'sitemaps', :action => 'index', :format => 'xml'
+
+        @models.each do |model|
+          expire_page :controller => 'sitemaps', :model => model, :format => 'xml' if !timestamps[model] || submit_time < timestamps[model]
+        end
+        SearchEngine.enabled.all.each do |search_engine|
+          if search_engine.submitted_at.nil? || search_engine.submitted_at < last_update
+            search_engine.submit
+          end
+        end
+      end
+    end
+
+    # Submit a single search engine. Called from run through the search engine
+    def submit(search_engine)
+      sumbmitter = new(search_engine)
+      sumbmitter.submit
+    end
+
+    # State what models to publish as a hash. 
+    # The keys are the plural names of the models.
+    # The values should be the scope to be used, formed as string
+    #    Cms::SitemapSubmitter.publish_models = {:pages => 'published.not_hidden', :news_articles => 'released' }
+    def publish_models=(models)
+      models.each_pair do |model, scope|
+        logger.debug "Backporting #{model} with bcms_sitemap accessors for selecting data - scope defined: '#{scope}'"
+        src = <<-end_src
+          class << self
+            def bcms_sitemap_scope
+              #{scope}.all
+            end
+            def bcms_sitemap_last_update
+              #{scope}.maximum(:updated_at)
+            end
+          end
+        end_src
+        model.to_s.classify.constantize.class_eval src, __FILE__, __LINE__
+      end
+      @models = models.keys.collect { |k| k.to_s  }.sort
+    end
+    
+    # the models defined by the application that will have sitemap information
+    def models
+      @models
+    end
+    
+    def logger #:nodoc:
+      RAILS_DEFAULT_LOGGER
+    end
+    
+    def perform_caching #:nodoc:
+      ApplicationController.perform_caching
+    end    
+  end
+  
+  
+  attr_accessor :search_engine, :connection
+  
+  
+  def initialize(search_engine) #:nodoc:
+    @search_engine = search_engine
+    @connection = ActiveResource::Connection.new(search_engine.url)
+  end
+  
+  def submit #:nodoc:
+    response = 200
+    begin
+      @connection.get(document_url)
+      logger.info "Sitemap was successfully submitted to #{search_engine.name} (#{document_url})"
+    rescue => e
+      logger.error "Sitemap submition failed for #{search_engine.name} (#{document_url})\nResponse was #{e.response}"
+      response = e.response
+    end
+    response
+  end
+  
+  def document_url #:nodoc:
+    @document_url ||= "#{search_engine.url}#{parameters}"
+  end
+  
+  def parameters #:nodoc:
+    CGI.escape "#{sitemaps_url(:host => SITE_DOMAIN)}"
+  end
+  
+  def logger #:nodoc:
+    self.class.logger
+  end
+end

data/lib/tasks/bcms_sitemap.rake

@@ -0,0 +1,4 @@
+unless defined? GEM_BCMS_SITEMAP
+  GEM_BCMS_SITEMAP = true
+  Dir["#{Gem.searcher.find('bcms_sitemap').full_gem_path}/lib/tasks/*.rake"].each { |ext| load ext }
+end

data/lib/tasks/sitemap.rake

@@ -0,0 +1,34 @@
+namespace :sitemap do  
+  desc 'Submit sitemap to relevant sites'
+  task :submit => :environment do
+    Cms::SitemapSubmitter.run
+  end
+  
+  desc 'Check status of sitemap submission'
+  task :status => :environment do
+    fmt = "%-20.20s    %1.1s    %3u %s\n"
+    printf fmt.gsub('u', 's'), 'Engine', 'Enabled', 'Sts', 'Last submit time'
+    puts '-' * 60
+    SearchEngine.all.each do |sitemap|
+      printf fmt, sitemap.name, (sitemap.enabled? ? 'Y' : 'N'), sitemap.last_status, sitemap.submitted_at
+    end
+  end
+  
+  desc <<-DESC
+    Verify that search engines verification files exists in pubic folder.
+    If they are not there, they will be created.
+    You want to run this as the last part of your deployment. If using capistrano you may add this to your deploy.rb file
+    
+    namespace :sitemap, :roles => :app do
+      task :verify_signatories do
+        rake = fetch(:rake, "rake")
+        rails_env = fetch(:rails_env, "production")
+        run "cd \#{current_path}; \#{rake} RAILS_ENV=\#{rails_env} sitemap:verify_signatories"
+      end
+    end
+    after "deploy:finalize_update", "sitemap:verify_signatories"
+  DESC
+  task :verify_signatories => :environment do
+    SearchEngine.verify_signatories!
+  end
+end

data/rails/init.rb

@@ -0,0 +1,6 @@
+gem_root = File.expand_path(File.join(File.dirname(__FILE__), ".."))
+Cms.add_to_rails_paths gem_root
+Cms.add_generator_paths gem_root, "db/migrate/[0-9]*_*.rb"
+Cms.add_generator_paths gem_root, "app/views/cms/shared/_admin_sidebar.html.erb"
+Cms.add_generator_paths gem_root, "config/initializers/bcms_sitemap.rb"
+Cms.add_generator_paths gem_root, "lib/tasks/bcms_sitemap.rake"

data/test/functional/sitemaps_controller_test.rb

@@ -0,0 +1,139 @@
+require 'test_helper'
+
+class SitemapsControllerTest < ActionController::TestCase  
+  # Replace this with your real tests.
+  
+  should_route :get, '/sitemaps.xml', :controller => :sitemaps, :action => :index
+  should_route :get, '/sitemaps/pages.xml', :controller => :sitemaps, :action => 'model', :model => 'pages'
+  should_route :get, '/sitemaps/news_articles.xml', :controller => :sitemaps, :action => 'model', :model => 'news_articles'
+  
+  XML_UTF8 = 'application/xml; charset=utf-8'
+  
+EXPECTED_SITEMAP = <<-TEXT
+<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <sitemap>
+    <loc>http://test.host/sitemaps/news_articles.xml</loc>
+    <lastmod>2010-02-05T10:54:49Z</lastmod>
+  </sitemap>
+  <sitemap>
+    <loc>http://test.host/sitemaps/pages.xml</loc>
+    <lastmod>2010-02-05T10:54:49Z</lastmod>
+  </sitemap>
+</sitemapindex>
+TEXT
+EXPECTED_EMPTY_SITEMAP = <<-TEXT
+<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+</sitemapindex>
+TEXT
+
+def page(path, priority = 0.9, frequency = :daily, archived = false)
+  @page = create_page(:path => path, :archived => archived)
+  @expected = <<-TEXT
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url>
+    <loc>http://localhost:3000#{path}</loc>
+    <lastmod>2010-02-05T10:54:49Z</lastmod>
+    <changefreq>#{frequency}</changefreq>
+    <priority>#{priority}</priority>
+  </url>
+</urlset>
+TEXT
+  end
+
+  
+  context 'sitemap' do
+    setup do
+      @request.env['Content-Type'] = 'application/xml'
+      @time = Time.parse '2010-02-05T10:54:49Z'
+      Time.stubs(:now).returns(@time)
+    end
+    context 'index' do
+      setup do
+        
+        get :index, :format => 'xml'
+      end
+      should 'return successfully' do
+        assert_response :success
+      end
+      should 'return correct format' do
+        assert_equal XML_UTF8, @response.headers['Content-Type']
+      end
+      should 'return correct structure and no content' do
+        assert_dom_equal EXPECTED_EMPTY_SITEMAP, @response.body
+      end
+
+      context 'having pages and news' do
+        setup do
+          @page = create_page 
+          @news = create_news_article
+          get :index
+        end
+        should 'return correct structure and content' do
+          assert_dom_equal EXPECTED_SITEMAP, @response.body
+        end
+      end
+    end
+    
+    context 'pages' do
+      setup do
+        get :model, :model => 'pages', :format => :xml
+      end
+      should 'generate map', :before => lambda { page('/')} do
+        assert_dom_equal @expected, @response.body
+      end
+      should 'rank home page', :before => lambda { page('/', 0.9, :daily)} do
+        assert_dom_equal @expected, @response.body
+      end
+      should 'rank navigation pages', :before => lambda { page('/nav/about', 0.4, :monthly)} do
+        assert_dom_equal @expected, @response.body
+      end
+      should 'rank news pages', :before => lambda { page('/news/articles', 0.7, :daily)} do
+        assert_dom_equal @expected, @response.body
+      end
+      should 'rank any non categorized pages', :before => lambda { page('/wow', 0.8, :weekly)} do
+        assert_dom_equal @expected, @response.body
+      end
+      should 'rank any archived pages', :before => lambda { page('/wow', 0.3, :never, true)} do
+        assert_dom_equal @expected, @response.body
+      end
+      should 'not include hidden pages', :before => lambda {
+        @page = create_page(:path => '/im/hidden', :hidden => true)
+      } do
+          @expected = <<-TEXT
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+</urlset>
+TEXT
+        assert_dom_equal @expected, @response.body
+      end
+    end
+    
+    context 'articles' do
+      setup do
+        @article = create_news_article(:name => 'This is the name of the article', :release_date => @time.to_date)
+      end
+      should_eventually 'publish articles' do
+        @expected = <<-TEXT
+        <?xml version="1.0" encoding="UTF-8"?>
+        <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+          <url>
+            <loc>http://localhost:3000/news/articles/2010/02/10/this-is-the-name-of-the-article</loc>
+            <lastmod>2010-02-05T10:54:49Z</lastmod>
+            <changefreq>weekly</changefreq>
+            <priority>0.7</priority>
+          </url>
+        </urlset>
+        TEXT
+        get :news_articles, :format => 'xml'
+        assert_response :success
+        assert_dom_equal @expected, @response.body
+      end
+      should_eventually 'give archived articles lower priority' do
+      end
+    end
+    
+  end  
+end

data/test/integration/sitemap_submitter_test.rb

@@ -0,0 +1,85 @@
+require 'test_helper'
+
+class Cms::SitemapSubmitterTest < ActiveSupport::TestCase
+  
+  context 'backporting models' do
+    should 'be initialized' do
+      assert Cms::SitemapSubmitter.models.is_a? Array
+      assert Cms::SitemapSubmitter.models.include? 'pages'
+      assert Cms::SitemapSubmitter.models.include? 'news_articles'
+    end
+    should 'just happen' do
+      assert Page.respond_to? :bcms_sitemap_scope
+      Cms::SitemapSubmitter.models.each do |model|
+        assert model.classify.constantize.respond_to? :bcms_sitemap_scope
+        assert model.classify.constantize.respond_to? :bcms_sitemap_last_update
+      end
+    end
+  end
+  
+  context 'submitting sitemap' do
+    setup do
+      @search_engine = create_search_engine
+      @submitter = Cms::SitemapSubmitter.new(@search_engine)
+    end
+    should 'return response code on success' do
+      @submitter.instance_variable_get(:@connection).stubs(:get).returns('')
+      resp = @submitter.submit
+      assert_equal 200, resp
+    end
+    should 'return response code on failure' do
+      @submitter.instance_variable_get(:@connection).expects(:get).raises(ActiveResource::ResourceNotFound, 404)
+      resp = @submitter.submit
+      assert_equal 404, resp
+    end
+    should 'generate correct parameters' do
+      assert_equal "http%3A%2F%2Flocalhost%3A3000%2Fsitemaps.xml", @submitter.parameters
+    end
+  end
+  
+  context 'run' do
+    setup do
+      @search_engine = create_search_engine
+      Cms::SitemapSubmitter.stubs(:submit).returns(200)
+    end
+    
+    should 'not submit if nothing updated' do
+      Cms::SitemapSubmitter.run
+      assert_nil @search_engine.reload.submitted_at
+    end
+
+    context 'based on updated' do
+      setup do
+        @search_engine.update_attributes(:submitted_at => Date.yesterday)
+        @last_time = @search_engine.submitted_at
+      end
+      should 'submit if never submitted' do
+        Cms::SitemapSubmitter.run
+        assert_not_nil @search_engine.reload.submitted_at
+      end
+      context 'pages' do
+        setup do
+          @page = create_page
+        end
+        should 'submit if something is updated since last time' do
+          Cms::SitemapSubmitter.run
+          assert_not_equal @last_time, @search_engine.reload.submitted_at
+        end
+        should 'update last_status' do
+          Cms::SitemapSubmitter.run
+          assert_equal 200, @search_engine.reload.last_status
+        end
+      end
+      context 'articles' do
+        setup do
+          @news = create_news_article
+        end
+        should 'submit if something is updated since last time' do
+          Cms::SitemapSubmitter.run
+          assert_not_equal @last_time, @search_engine.reload.submitted_at
+        end        
+      end
+    end
+  end
+  
+end

data/test/performance/browsing_test.rb

@@ -0,0 +1,9 @@
+require 'test_helper'
+require 'performance_test_help'
+
+# Profiling results for each test method are written to tmp/performance.
+class BrowsingTest < ActionController::PerformanceTest
+  def test_homepage
+    get '/'
+  end
+end