bcms_sitemap 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,4 @@
1
+ # State what models to publish as a hash.
2
+ # The keys are the plural names of the models.
3
+ # The values should be the scope to be used, formed as string
4
+ Cms::SitemapSubmitter.publish_models = {:pages => 'published.not_hidden', :news_articles => 'released'}
@@ -0,0 +1 @@
1
+ require File.join(File.dirname(__FILE__), "..", "..", "lib", "bcms_sitemap")
@@ -0,0 +1,24 @@
1
+ class CreateSearchEngines < ActiveRecord::Migration
2
+ def self.up
3
+ create_table :search_engines do |t|
4
+ t.string :name
5
+ t.boolean :enabled
6
+ t.string :url
7
+ t.string :verification_file
8
+ t.text :verification_content
9
+ t.integer :last_status
10
+ t.timestamp :submitted_at
11
+ t.timestamps
12
+ end
13
+ # Create a sample search engine
14
+ SearchEngine.create :name => 'Google', :url => 'http://www.google.com/webmasters/tools/ping?sitemap='
15
+ SearchEngine.create :name => 'Yahoo', :url => 'http://search.yahooapis.com/SiteExplorerService/V1/ping?sitemap='
16
+ SearchEngine.create :name => 'Ask', :url => 'http://submissions.ask.com/ping?sitemap='
17
+ SearchEngine.create :name => 'Live/Bing', :url => 'http://www.bing.com/webmaster/ping.aspx?siteMap='
18
+ SearchEngine.create :name => 'Moreover', :url => 'http://api.moreover.com/ping?u='
19
+ end
20
+
21
+ def self.down
22
+ drop_table :search_engines
23
+ end
24
+ end
@@ -0,0 +1,3 @@
1
+ require 'bcms_sitemap/routes'
2
+ require 'bcms_sitemap/sitemap_submitter'
3
+ require File.join(File.dirname(__FILE__), '..', 'app', 'controllers', 'sitemaps_controller')
@@ -0,0 +1,10 @@
1
+ module Cms::Routes
2
+ def routes_for_bcms_sitemap
3
+ sitemaps 'sitemaps.xml', :controller => :sitemaps, :action => 'index'
4
+ sitemap_for_model 'sitemaps/:model.xml', :controller => :sitemaps, :action => 'model',
5
+ :requirements => { :model => /#{Cms::SitemapSubmitter.models.join('|')}/ }
6
+ namespace :cms do |cms|
7
+ cms.resources :search_engines
8
+ end
9
+ end
10
+ end
@@ -0,0 +1,110 @@
1
+ require 'cgi'
2
+ class Cms::SitemapSubmitter
3
+ include ActionController::UrlWriter
4
+ include ActionController::Caching::Pages
5
+
6
+ class << self
7
+
8
+ # Checks to see if there has been any updates since last time.
9
+ # If so, clears the cache for the relevant model and submits the url to the search engine's that have not yet been notified
10
+ def run
11
+ submit_time = SearchEngine.enabled.minimum(:submitted_at)
12
+
13
+ # collect timestamp for all models. We don't want to expire more pages than necessary
14
+ timestamps = {}
15
+ @models.each do |model|
16
+ last_update = model.classify.constantize.bcms_sitemap_last_update
17
+ timestamps[model] = last_update if last_update
18
+ end
19
+ last_update = timestamps.values.compact.max
20
+ # try this {}.values.compact.max
21
+ if last_update && (submit_time.nil? || submit_time < last_update)
22
+ # This is a lazy cleaning of cache
23
+ expire_page :controller => 'sitemaps', :action => 'index', :format => 'xml'
24
+
25
+ @models.each do |model|
26
+ expire_page :controller => 'sitemaps', :model => model, :format => 'xml' if !timestamps[model] || submit_time < timestamps[model]
27
+ end
28
+ SearchEngine.enabled.all.each do |search_engine|
29
+ if search_engine.submitted_at.nil? || search_engine.submitted_at < last_update
30
+ search_engine.submit
31
+ end
32
+ end
33
+ end
34
+ end
35
+
36
+ # Submit a single search engine. Called from run through the search engine
37
+ def submit(search_engine)
38
+ sumbmitter = new(search_engine)
39
+ sumbmitter.submit
40
+ end
41
+
42
+ # State what models to publish as a hash.
43
+ # The keys are the plural names of the models.
44
+ # The values should be the scope to be used, formed as string
45
+ # Cms::SitemapSubmitter.publish_models = {:pages => 'published.not_hidden', :news_articles => 'released' }
46
+ def publish_models=(models)
47
+ models.each_pair do |model, scope|
48
+ logger.debug "Backporting #{model} with bcms_sitemap accessors for selecting data - scope defined: '#{scope}'"
49
+ src = <<-end_src
50
+ class << self
51
+ def bcms_sitemap_scope
52
+ #{scope}.all
53
+ end
54
+ def bcms_sitemap_last_update
55
+ #{scope}.maximum(:updated_at)
56
+ end
57
+ end
58
+ end_src
59
+ model.to_s.classify.constantize.class_eval src, __FILE__, __LINE__
60
+ end
61
+ @models = models.keys.collect { |k| k.to_s }.sort
62
+ end
63
+
64
+ # the models defined by the application that will have sitemap information
65
+ def models
66
+ @models
67
+ end
68
+
69
+ def logger #:nodoc:
70
+ RAILS_DEFAULT_LOGGER
71
+ end
72
+
73
+ def perform_caching #:nodoc:
74
+ ApplicationController.perform_caching
75
+ end
76
+ end
77
+
78
+
79
+ attr_accessor :search_engine, :connection
80
+
81
+
82
+ def initialize(search_engine) #:nodoc:
83
+ @search_engine = search_engine
84
+ @connection = ActiveResource::Connection.new(search_engine.url)
85
+ end
86
+
87
+ def submit #:nodoc:
88
+ response = 200
89
+ begin
90
+ @connection.get(document_url)
91
+ logger.info "Sitemap was successfully submitted to #{search_engine.name} (#{document_url})"
92
+ rescue => e
93
+ logger.error "Sitemap submition failed for #{search_engine.name} (#{document_url})\nResponse was #{e.response}"
94
+ response = e.response
95
+ end
96
+ response
97
+ end
98
+
99
+ def document_url #:nodoc:
100
+ @document_url ||= "#{search_engine.url}#{parameters}"
101
+ end
102
+
103
+ def parameters #:nodoc:
104
+ CGI.escape "#{sitemaps_url(:host => SITE_DOMAIN)}"
105
+ end
106
+
107
+ def logger #:nodoc:
108
+ self.class.logger
109
+ end
110
+ end
@@ -0,0 +1,4 @@
1
+ unless defined? GEM_BCMS_SITEMAP
2
+ GEM_BCMS_SITEMAP = true
3
+ Dir["#{Gem.searcher.find('bcms_sitemap').full_gem_path}/lib/tasks/*.rake"].each { |ext| load ext }
4
+ end
@@ -0,0 +1,34 @@
1
+ namespace :sitemap do
2
+ desc 'Submit sitemap to relevant sites'
3
+ task :submit => :environment do
4
+ Cms::SitemapSubmitter.run
5
+ end
6
+
7
+ desc 'Check status of sitemap submission'
8
+ task :status => :environment do
9
+ fmt = "%-20.20s %1.1s %3u %s\n"
10
+ printf fmt.gsub('u', 's'), 'Engine', 'Enabled', 'Sts', 'Last submit time'
11
+ puts '-' * 60
12
+ SearchEngine.all.each do |sitemap|
13
+ printf fmt, sitemap.name, (sitemap.enabled? ? 'Y' : 'N'), sitemap.last_status, sitemap.submitted_at
14
+ end
15
+ end
16
+
17
+ desc <<-DESC
18
+ Verify that search engines verification files exists in pubic folder.
19
+ If they are not there, they will be created.
20
+ You want to run this as the last part of your deployment. If using capistrano you may add this to your deploy.rb file
21
+
22
+ namespace :sitemap, :roles => :app do
23
+ task :verify_signatories do
24
+ rake = fetch(:rake, "rake")
25
+ rails_env = fetch(:rails_env, "production")
26
+ run "cd \#{current_path}; \#{rake} RAILS_ENV=\#{rails_env} sitemap:verify_signatories"
27
+ end
28
+ end
29
+ after "deploy:finalize_update", "sitemap:verify_signatories"
30
+ DESC
31
+ task :verify_signatories => :environment do
32
+ SearchEngine.verify_signatories!
33
+ end
34
+ end
data/rails/init.rb ADDED
@@ -0,0 +1,6 @@
1
+ gem_root = File.expand_path(File.join(File.dirname(__FILE__), ".."))
2
+ Cms.add_to_rails_paths gem_root
3
+ Cms.add_generator_paths gem_root, "db/migrate/[0-9]*_*.rb"
4
+ Cms.add_generator_paths gem_root, "app/views/cms/shared/_admin_sidebar.html.erb"
5
+ Cms.add_generator_paths gem_root, "config/initializers/bcms_sitemap.rb"
6
+ Cms.add_generator_paths gem_root, "lib/tasks/bcms_sitemap.rake"
@@ -0,0 +1,139 @@
1
+ require 'test_helper'
2
+
3
+ class SitemapsControllerTest < ActionController::TestCase
4
+ # Replace this with your real tests.
5
+
6
+ should_route :get, '/sitemaps.xml', :controller => :sitemaps, :action => :index
7
+ should_route :get, '/sitemaps/pages.xml', :controller => :sitemaps, :action => 'model', :model => 'pages'
8
+ should_route :get, '/sitemaps/news_articles.xml', :controller => :sitemaps, :action => 'model', :model => 'news_articles'
9
+
10
+ XML_UTF8 = 'application/xml; charset=utf-8'
11
+
12
+ EXPECTED_SITEMAP = <<-TEXT
13
+ <?xml version="1.0" encoding="UTF-8"?>
14
+ <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
15
+ <sitemap>
16
+ <loc>http://test.host/sitemaps/news_articles.xml</loc>
17
+ <lastmod>2010-02-05T10:54:49Z</lastmod>
18
+ </sitemap>
19
+ <sitemap>
20
+ <loc>http://test.host/sitemaps/pages.xml</loc>
21
+ <lastmod>2010-02-05T10:54:49Z</lastmod>
22
+ </sitemap>
23
+ </sitemapindex>
24
+ TEXT
25
+ EXPECTED_EMPTY_SITEMAP = <<-TEXT
26
+ <?xml version="1.0" encoding="UTF-8"?>
27
+ <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
28
+ </sitemapindex>
29
+ TEXT
30
+
31
+ def page(path, priority = 0.9, frequency = :daily, archived = false)
32
+ @page = create_page(:path => path, :archived => archived)
33
+ @expected = <<-TEXT
34
+ <?xml version="1.0" encoding="UTF-8"?>
35
+ <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
36
+ <url>
37
+ <loc>http://localhost:3000#{path}</loc>
38
+ <lastmod>2010-02-05T10:54:49Z</lastmod>
39
+ <changefreq>#{frequency}</changefreq>
40
+ <priority>#{priority}</priority>
41
+ </url>
42
+ </urlset>
43
+ TEXT
44
+ end
45
+
46
+
47
+ context 'sitemap' do
48
+ setup do
49
+ @request.env['Content-Type'] = 'application/xml'
50
+ @time = Time.parse '2010-02-05T10:54:49Z'
51
+ Time.stubs(:now).returns(@time)
52
+ end
53
+ context 'index' do
54
+ setup do
55
+
56
+ get :index, :format => 'xml'
57
+ end
58
+ should 'return successfully' do
59
+ assert_response :success
60
+ end
61
+ should 'return correct format' do
62
+ assert_equal XML_UTF8, @response.headers['Content-Type']
63
+ end
64
+ should 'return correct structure and no content' do
65
+ assert_dom_equal EXPECTED_EMPTY_SITEMAP, @response.body
66
+ end
67
+
68
+ context 'having pages and news' do
69
+ setup do
70
+ @page = create_page
71
+ @news = create_news_article
72
+ get :index
73
+ end
74
+ should 'return correct structure and content' do
75
+ assert_dom_equal EXPECTED_SITEMAP, @response.body
76
+ end
77
+ end
78
+ end
79
+
80
+ context 'pages' do
81
+ setup do
82
+ get :model, :model => 'pages', :format => :xml
83
+ end
84
+ should 'generate map', :before => lambda { page('/')} do
85
+ assert_dom_equal @expected, @response.body
86
+ end
87
+ should 'rank home page', :before => lambda { page('/', 0.9, :daily)} do
88
+ assert_dom_equal @expected, @response.body
89
+ end
90
+ should 'rank navigation pages', :before => lambda { page('/nav/about', 0.4, :monthly)} do
91
+ assert_dom_equal @expected, @response.body
92
+ end
93
+ should 'rank news pages', :before => lambda { page('/news/articles', 0.7, :daily)} do
94
+ assert_dom_equal @expected, @response.body
95
+ end
96
+ should 'rank any non categorized pages', :before => lambda { page('/wow', 0.8, :weekly)} do
97
+ assert_dom_equal @expected, @response.body
98
+ end
99
+ should 'rank any archived pages', :before => lambda { page('/wow', 0.3, :never, true)} do
100
+ assert_dom_equal @expected, @response.body
101
+ end
102
+ should 'not include hidden pages', :before => lambda {
103
+ @page = create_page(:path => '/im/hidden', :hidden => true)
104
+ } do
105
+ @expected = <<-TEXT
106
+ <?xml version="1.0" encoding="UTF-8"?>
107
+ <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
108
+ </urlset>
109
+ TEXT
110
+ assert_dom_equal @expected, @response.body
111
+ end
112
+ end
113
+
114
+ context 'articles' do
115
+ setup do
116
+ @article = create_news_article(:name => 'This is the name of the article', :release_date => @time.to_date)
117
+ end
118
+ should_eventually 'publish articles' do
119
+ @expected = <<-TEXT
120
+ <?xml version="1.0" encoding="UTF-8"?>
121
+ <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
122
+ <url>
123
+ <loc>http://localhost:3000/news/articles/2010/02/10/this-is-the-name-of-the-article</loc>
124
+ <lastmod>2010-02-05T10:54:49Z</lastmod>
125
+ <changefreq>weekly</changefreq>
126
+ <priority>0.7</priority>
127
+ </url>
128
+ </urlset>
129
+ TEXT
130
+ get :news_articles, :format => 'xml'
131
+ assert_response :success
132
+ assert_dom_equal @expected, @response.body
133
+ end
134
+ should_eventually 'give archived articles lower priority' do
135
+ end
136
+ end
137
+
138
+ end
139
+ end
@@ -0,0 +1,85 @@
1
+ require 'test_helper'
2
+
3
+ class Cms::SitemapSubmitterTest < ActiveSupport::TestCase
4
+
5
+ context 'backporting models' do
6
+ should 'be initialized' do
7
+ assert Cms::SitemapSubmitter.models.is_a? Array
8
+ assert Cms::SitemapSubmitter.models.include? 'pages'
9
+ assert Cms::SitemapSubmitter.models.include? 'news_articles'
10
+ end
11
+ should 'just happen' do
12
+ assert Page.respond_to? :bcms_sitemap_scope
13
+ Cms::SitemapSubmitter.models.each do |model|
14
+ assert model.classify.constantize.respond_to? :bcms_sitemap_scope
15
+ assert model.classify.constantize.respond_to? :bcms_sitemap_last_update
16
+ end
17
+ end
18
+ end
19
+
20
+ context 'submitting sitemap' do
21
+ setup do
22
+ @search_engine = create_search_engine
23
+ @submitter = Cms::SitemapSubmitter.new(@search_engine)
24
+ end
25
+ should 'return response code on success' do
26
+ @submitter.instance_variable_get(:@connection).stubs(:get).returns('')
27
+ resp = @submitter.submit
28
+ assert_equal 200, resp
29
+ end
30
+ should 'return response code on failure' do
31
+ @submitter.instance_variable_get(:@connection).expects(:get).raises(ActiveResource::ResourceNotFound, 404)
32
+ resp = @submitter.submit
33
+ assert_equal 404, resp
34
+ end
35
+ should 'generate correct parameters' do
36
+ assert_equal "http%3A%2F%2Flocalhost%3A3000%2Fsitemaps.xml", @submitter.parameters
37
+ end
38
+ end
39
+
40
+ context 'run' do
41
+ setup do
42
+ @search_engine = create_search_engine
43
+ Cms::SitemapSubmitter.stubs(:submit).returns(200)
44
+ end
45
+
46
+ should 'not submit if nothing updated' do
47
+ Cms::SitemapSubmitter.run
48
+ assert_nil @search_engine.reload.submitted_at
49
+ end
50
+
51
+ context 'based on updated' do
52
+ setup do
53
+ @search_engine.update_attributes(:submitted_at => Date.yesterday)
54
+ @last_time = @search_engine.submitted_at
55
+ end
56
+ should 'submit if never submitted' do
57
+ Cms::SitemapSubmitter.run
58
+ assert_not_nil @search_engine.reload.submitted_at
59
+ end
60
+ context 'pages' do
61
+ setup do
62
+ @page = create_page
63
+ end
64
+ should 'submit if something is updated since last time' do
65
+ Cms::SitemapSubmitter.run
66
+ assert_not_equal @last_time, @search_engine.reload.submitted_at
67
+ end
68
+ should 'update last_status' do
69
+ Cms::SitemapSubmitter.run
70
+ assert_equal 200, @search_engine.reload.last_status
71
+ end
72
+ end
73
+ context 'articles' do
74
+ setup do
75
+ @news = create_news_article
76
+ end
77
+ should 'submit if something is updated since last time' do
78
+ Cms::SitemapSubmitter.run
79
+ assert_not_equal @last_time, @search_engine.reload.submitted_at
80
+ end
81
+ end
82
+ end
83
+ end
84
+
85
+ end
@@ -0,0 +1,9 @@
1
+ require 'test_helper'
2
+ require 'performance_test_help'
3
+
4
+ # Profiling results for each test method are written to tmp/performance.
5
+ class BrowsingTest < ActionController::PerformanceTest
6
+ def test_homepage
7
+ get '/'
8
+ end
9
+ end