bcms_sitemap 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/LICENSE.txt +674 -0
- data/README.markdown +167 -0
- data/app/controllers/cms/search_engines_controller.rb +19 -0
- data/app/controllers/sitemaps_controller.rb +22 -0
- data/app/helpers/sitemaps_helper.rb +13 -0
- data/app/models/search_engine.rb +69 -0
- data/app/views/cms/search_engines/_form.html.erb +17 -0
- data/app/views/cms/search_engines/edit.html.erb +3 -0
- data/app/views/cms/search_engines/index.html.erb +51 -0
- data/app/views/cms/search_engines/new.html.erb +3 -0
- data/app/views/cms/shared/_admin_sidebar.html.erb +31 -0
- data/app/views/sitemaps/index.builder +10 -0
- data/app/views/sitemaps/model.builder +11 -0
- data/app/views/sitemaps/news_articles.builder +11 -0
- data/config/initializers/bcms_sitemap.rb +4 -0
- data/config/initializers/init_module.rb +1 -0
- data/db/migrate/20100212083005_create_search_engines.rb +24 -0
- data/lib/bcms_sitemap.rb +3 -0
- data/lib/bcms_sitemap/routes.rb +10 -0
- data/lib/bcms_sitemap/sitemap_submitter.rb +110 -0
- data/lib/tasks/bcms_sitemap.rake +4 -0
- data/lib/tasks/sitemap.rake +34 -0
- data/rails/init.rb +6 -0
- data/test/functional/sitemaps_controller_test.rb +139 -0
- data/test/integration/sitemap_submitter_test.rb +85 -0
- data/test/performance/browsing_test.rb +9 -0
- data/test/test_factory.rb +29 -0
- data/test/test_helper.rb +39 -0
- data/test/unit/helpers/sitemaps_helper_test.rb +4 -0
- data/test/unit/search_engine_test.rb +126 -0
- metadata +93 -0
@@ -0,0 +1 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__), "..", "..", "lib", "bcms_sitemap")
|
@@ -0,0 +1,24 @@
|
|
1
|
+
class CreateSearchEngines < ActiveRecord::Migration
|
2
|
+
def self.up
|
3
|
+
create_table :search_engines do |t|
|
4
|
+
t.string :name
|
5
|
+
t.boolean :enabled
|
6
|
+
t.string :url
|
7
|
+
t.string :verification_file
|
8
|
+
t.text :verification_content
|
9
|
+
t.integer :last_status
|
10
|
+
t.timestamp :submitted_at
|
11
|
+
t.timestamps
|
12
|
+
end
|
13
|
+
# Create a sample search engine
|
14
|
+
SearchEngine.create :name => 'Google', :url => 'http://www.google.com/webmasters/tools/ping?sitemap='
|
15
|
+
SearchEngine.create :name => 'Yahoo', :url => 'http://search.yahooapis.com/SiteExplorerService/V1/ping?sitemap='
|
16
|
+
SearchEngine.create :name => 'Ask', :url => 'http://submissions.ask.com/ping?sitemap='
|
17
|
+
SearchEngine.create :name => 'Live/Bing', :url => 'http://www.bing.com/webmaster/ping.aspx?siteMap='
|
18
|
+
SearchEngine.create :name => 'Moreover', :url => 'http://api.moreover.com/ping?u='
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.down
|
22
|
+
drop_table :search_engines
|
23
|
+
end
|
24
|
+
end
|
data/lib/bcms_sitemap.rb
ADDED
@@ -0,0 +1,10 @@
|
|
1
|
+
module Cms::Routes
|
2
|
+
def routes_for_bcms_sitemap
|
3
|
+
sitemaps 'sitemaps.xml', :controller => :sitemaps, :action => 'index'
|
4
|
+
sitemap_for_model 'sitemaps/:model.xml', :controller => :sitemaps, :action => 'model',
|
5
|
+
:requirements => { :model => /#{Cms::SitemapSubmitter.models.join('|')}/ }
|
6
|
+
namespace :cms do |cms|
|
7
|
+
cms.resources :search_engines
|
8
|
+
end
|
9
|
+
end
|
10
|
+
end
|
@@ -0,0 +1,110 @@
|
|
1
|
+
require 'cgi'
|
2
|
+
class Cms::SitemapSubmitter
|
3
|
+
include ActionController::UrlWriter
|
4
|
+
include ActionController::Caching::Pages
|
5
|
+
|
6
|
+
class << self
|
7
|
+
|
8
|
+
# Checks to see if there has been any updates since last time.
|
9
|
+
# If so, clears the cache for the relevant model and submits the url to the search engine's that have not yet been notified
|
10
|
+
def run
|
11
|
+
submit_time = SearchEngine.enabled.minimum(:submitted_at)
|
12
|
+
|
13
|
+
# collect timestamp for all models. We don't want to expire more pages than necessary
|
14
|
+
timestamps = {}
|
15
|
+
@models.each do |model|
|
16
|
+
last_update = model.classify.constantize.bcms_sitemap_last_update
|
17
|
+
timestamps[model] = last_update if last_update
|
18
|
+
end
|
19
|
+
last_update = timestamps.values.compact.max
|
20
|
+
# try this {}.values.compact.max
|
21
|
+
if last_update && (submit_time.nil? || submit_time < last_update)
|
22
|
+
# This is a lazy cleaning of cache
|
23
|
+
expire_page :controller => 'sitemaps', :action => 'index', :format => 'xml'
|
24
|
+
|
25
|
+
@models.each do |model|
|
26
|
+
expire_page :controller => 'sitemaps', :model => model, :format => 'xml' if !timestamps[model] || submit_time < timestamps[model]
|
27
|
+
end
|
28
|
+
SearchEngine.enabled.all.each do |search_engine|
|
29
|
+
if search_engine.submitted_at.nil? || search_engine.submitted_at < last_update
|
30
|
+
search_engine.submit
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
# Submit a single search engine. Called from run through the search engine
|
37
|
+
def submit(search_engine)
|
38
|
+
sumbmitter = new(search_engine)
|
39
|
+
sumbmitter.submit
|
40
|
+
end
|
41
|
+
|
42
|
+
# State what models to publish as a hash.
|
43
|
+
# The keys are the plural names of the models.
|
44
|
+
# The values should be the scope to be used, formed as string
|
45
|
+
# Cms::SitemapSubmitter.publish_models = {:pages => 'published.not_hidden', :news_articles => 'released' }
|
46
|
+
def publish_models=(models)
|
47
|
+
models.each_pair do |model, scope|
|
48
|
+
logger.debug "Backporting #{model} with bcms_sitemap accessors for selecting data - scope defined: '#{scope}'"
|
49
|
+
src = <<-end_src
|
50
|
+
class << self
|
51
|
+
def bcms_sitemap_scope
|
52
|
+
#{scope}.all
|
53
|
+
end
|
54
|
+
def bcms_sitemap_last_update
|
55
|
+
#{scope}.maximum(:updated_at)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end_src
|
59
|
+
model.to_s.classify.constantize.class_eval src, __FILE__, __LINE__
|
60
|
+
end
|
61
|
+
@models = models.keys.collect { |k| k.to_s }.sort
|
62
|
+
end
|
63
|
+
|
64
|
+
# the models defined by the application that will have sitemap information
|
65
|
+
def models
|
66
|
+
@models
|
67
|
+
end
|
68
|
+
|
69
|
+
def logger #:nodoc:
|
70
|
+
RAILS_DEFAULT_LOGGER
|
71
|
+
end
|
72
|
+
|
73
|
+
def perform_caching #:nodoc:
|
74
|
+
ApplicationController.perform_caching
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
|
79
|
+
attr_accessor :search_engine, :connection
|
80
|
+
|
81
|
+
|
82
|
+
def initialize(search_engine) #:nodoc:
|
83
|
+
@search_engine = search_engine
|
84
|
+
@connection = ActiveResource::Connection.new(search_engine.url)
|
85
|
+
end
|
86
|
+
|
87
|
+
def submit #:nodoc:
|
88
|
+
response = 200
|
89
|
+
begin
|
90
|
+
@connection.get(document_url)
|
91
|
+
logger.info "Sitemap was successfully submitted to #{search_engine.name} (#{document_url})"
|
92
|
+
rescue => e
|
93
|
+
logger.error "Sitemap submition failed for #{search_engine.name} (#{document_url})\nResponse was #{e.response}"
|
94
|
+
response = e.response
|
95
|
+
end
|
96
|
+
response
|
97
|
+
end
|
98
|
+
|
99
|
+
def document_url #:nodoc:
|
100
|
+
@document_url ||= "#{search_engine.url}#{parameters}"
|
101
|
+
end
|
102
|
+
|
103
|
+
def parameters #:nodoc:
|
104
|
+
CGI.escape "#{sitemaps_url(:host => SITE_DOMAIN)}"
|
105
|
+
end
|
106
|
+
|
107
|
+
def logger #:nodoc:
|
108
|
+
self.class.logger
|
109
|
+
end
|
110
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
namespace :sitemap do
|
2
|
+
desc 'Submit sitemap to relevant sites'
|
3
|
+
task :submit => :environment do
|
4
|
+
Cms::SitemapSubmitter.run
|
5
|
+
end
|
6
|
+
|
7
|
+
desc 'Check status of sitemap submission'
|
8
|
+
task :status => :environment do
|
9
|
+
fmt = "%-20.20s %1.1s %3u %s\n"
|
10
|
+
printf fmt.gsub('u', 's'), 'Engine', 'Enabled', 'Sts', 'Last submit time'
|
11
|
+
puts '-' * 60
|
12
|
+
SearchEngine.all.each do |sitemap|
|
13
|
+
printf fmt, sitemap.name, (sitemap.enabled? ? 'Y' : 'N'), sitemap.last_status, sitemap.submitted_at
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
desc <<-DESC
|
18
|
+
Verify that search engines verification files exists in pubic folder.
|
19
|
+
If they are not there, they will be created.
|
20
|
+
You want to run this as the last part of your deployment. If using capistrano you may add this to your deploy.rb file
|
21
|
+
|
22
|
+
namespace :sitemap, :roles => :app do
|
23
|
+
task :verify_signatories do
|
24
|
+
rake = fetch(:rake, "rake")
|
25
|
+
rails_env = fetch(:rails_env, "production")
|
26
|
+
run "cd \#{current_path}; \#{rake} RAILS_ENV=\#{rails_env} sitemap:verify_signatories"
|
27
|
+
end
|
28
|
+
end
|
29
|
+
after "deploy:finalize_update", "sitemap:verify_signatories"
|
30
|
+
DESC
|
31
|
+
task :verify_signatories => :environment do
|
32
|
+
SearchEngine.verify_signatories!
|
33
|
+
end
|
34
|
+
end
|
data/rails/init.rb
ADDED
@@ -0,0 +1,6 @@
|
|
1
|
+
gem_root = File.expand_path(File.join(File.dirname(__FILE__), ".."))
|
2
|
+
Cms.add_to_rails_paths gem_root
|
3
|
+
Cms.add_generator_paths gem_root, "db/migrate/[0-9]*_*.rb"
|
4
|
+
Cms.add_generator_paths gem_root, "app/views/cms/shared/_admin_sidebar.html.erb"
|
5
|
+
Cms.add_generator_paths gem_root, "config/initializers/bcms_sitemap.rb"
|
6
|
+
Cms.add_generator_paths gem_root, "lib/tasks/bcms_sitemap.rake"
|
@@ -0,0 +1,139 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
class SitemapsControllerTest < ActionController::TestCase
|
4
|
+
# Replace this with your real tests.
|
5
|
+
|
6
|
+
should_route :get, '/sitemaps.xml', :controller => :sitemaps, :action => :index
|
7
|
+
should_route :get, '/sitemaps/pages.xml', :controller => :sitemaps, :action => 'model', :model => 'pages'
|
8
|
+
should_route :get, '/sitemaps/news_articles.xml', :controller => :sitemaps, :action => 'model', :model => 'news_articles'
|
9
|
+
|
10
|
+
XML_UTF8 = 'application/xml; charset=utf-8'
|
11
|
+
|
12
|
+
EXPECTED_SITEMAP = <<-TEXT
|
13
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
14
|
+
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
|
15
|
+
<sitemap>
|
16
|
+
<loc>http://test.host/sitemaps/news_articles.xml</loc>
|
17
|
+
<lastmod>2010-02-05T10:54:49Z</lastmod>
|
18
|
+
</sitemap>
|
19
|
+
<sitemap>
|
20
|
+
<loc>http://test.host/sitemaps/pages.xml</loc>
|
21
|
+
<lastmod>2010-02-05T10:54:49Z</lastmod>
|
22
|
+
</sitemap>
|
23
|
+
</sitemapindex>
|
24
|
+
TEXT
|
25
|
+
EXPECTED_EMPTY_SITEMAP = <<-TEXT
|
26
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
27
|
+
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
|
28
|
+
</sitemapindex>
|
29
|
+
TEXT
|
30
|
+
|
31
|
+
def page(path, priority = 0.9, frequency = :daily, archived = false)
|
32
|
+
@page = create_page(:path => path, :archived => archived)
|
33
|
+
@expected = <<-TEXT
|
34
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
35
|
+
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
|
36
|
+
<url>
|
37
|
+
<loc>http://localhost:3000#{path}</loc>
|
38
|
+
<lastmod>2010-02-05T10:54:49Z</lastmod>
|
39
|
+
<changefreq>#{frequency}</changefreq>
|
40
|
+
<priority>#{priority}</priority>
|
41
|
+
</url>
|
42
|
+
</urlset>
|
43
|
+
TEXT
|
44
|
+
end
|
45
|
+
|
46
|
+
|
47
|
+
context 'sitemap' do
|
48
|
+
setup do
|
49
|
+
@request.env['Content-Type'] = 'application/xml'
|
50
|
+
@time = Time.parse '2010-02-05T10:54:49Z'
|
51
|
+
Time.stubs(:now).returns(@time)
|
52
|
+
end
|
53
|
+
context 'index' do
|
54
|
+
setup do
|
55
|
+
|
56
|
+
get :index, :format => 'xml'
|
57
|
+
end
|
58
|
+
should 'return successfully' do
|
59
|
+
assert_response :success
|
60
|
+
end
|
61
|
+
should 'return correct format' do
|
62
|
+
assert_equal XML_UTF8, @response.headers['Content-Type']
|
63
|
+
end
|
64
|
+
should 'return correct structure and no content' do
|
65
|
+
assert_dom_equal EXPECTED_EMPTY_SITEMAP, @response.body
|
66
|
+
end
|
67
|
+
|
68
|
+
context 'having pages and news' do
|
69
|
+
setup do
|
70
|
+
@page = create_page
|
71
|
+
@news = create_news_article
|
72
|
+
get :index
|
73
|
+
end
|
74
|
+
should 'return correct structure and content' do
|
75
|
+
assert_dom_equal EXPECTED_SITEMAP, @response.body
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
context 'pages' do
|
81
|
+
setup do
|
82
|
+
get :model, :model => 'pages', :format => :xml
|
83
|
+
end
|
84
|
+
should 'generate map', :before => lambda { page('/')} do
|
85
|
+
assert_dom_equal @expected, @response.body
|
86
|
+
end
|
87
|
+
should 'rank home page', :before => lambda { page('/', 0.9, :daily)} do
|
88
|
+
assert_dom_equal @expected, @response.body
|
89
|
+
end
|
90
|
+
should 'rank navigation pages', :before => lambda { page('/nav/about', 0.4, :monthly)} do
|
91
|
+
assert_dom_equal @expected, @response.body
|
92
|
+
end
|
93
|
+
should 'rank news pages', :before => lambda { page('/news/articles', 0.7, :daily)} do
|
94
|
+
assert_dom_equal @expected, @response.body
|
95
|
+
end
|
96
|
+
should 'rank any non categorized pages', :before => lambda { page('/wow', 0.8, :weekly)} do
|
97
|
+
assert_dom_equal @expected, @response.body
|
98
|
+
end
|
99
|
+
should 'rank any archived pages', :before => lambda { page('/wow', 0.3, :never, true)} do
|
100
|
+
assert_dom_equal @expected, @response.body
|
101
|
+
end
|
102
|
+
should 'not include hidden pages', :before => lambda {
|
103
|
+
@page = create_page(:path => '/im/hidden', :hidden => true)
|
104
|
+
} do
|
105
|
+
@expected = <<-TEXT
|
106
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
107
|
+
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
|
108
|
+
</urlset>
|
109
|
+
TEXT
|
110
|
+
assert_dom_equal @expected, @response.body
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
context 'articles' do
|
115
|
+
setup do
|
116
|
+
@article = create_news_article(:name => 'This is the name of the article', :release_date => @time.to_date)
|
117
|
+
end
|
118
|
+
should_eventually 'publish articles' do
|
119
|
+
@expected = <<-TEXT
|
120
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
121
|
+
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
|
122
|
+
<url>
|
123
|
+
<loc>http://localhost:3000/news/articles/2010/02/10/this-is-the-name-of-the-article</loc>
|
124
|
+
<lastmod>2010-02-05T10:54:49Z</lastmod>
|
125
|
+
<changefreq>weekly</changefreq>
|
126
|
+
<priority>0.7</priority>
|
127
|
+
</url>
|
128
|
+
</urlset>
|
129
|
+
TEXT
|
130
|
+
get :news_articles, :format => 'xml'
|
131
|
+
assert_response :success
|
132
|
+
assert_dom_equal @expected, @response.body
|
133
|
+
end
|
134
|
+
should_eventually 'give archived articles lower priority' do
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
end
|
139
|
+
end
|
@@ -0,0 +1,85 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
class Cms::SitemapSubmitterTest < ActiveSupport::TestCase
|
4
|
+
|
5
|
+
context 'backporting models' do
|
6
|
+
should 'be initialized' do
|
7
|
+
assert Cms::SitemapSubmitter.models.is_a? Array
|
8
|
+
assert Cms::SitemapSubmitter.models.include? 'pages'
|
9
|
+
assert Cms::SitemapSubmitter.models.include? 'news_articles'
|
10
|
+
end
|
11
|
+
should 'just happen' do
|
12
|
+
assert Page.respond_to? :bcms_sitemap_scope
|
13
|
+
Cms::SitemapSubmitter.models.each do |model|
|
14
|
+
assert model.classify.constantize.respond_to? :bcms_sitemap_scope
|
15
|
+
assert model.classify.constantize.respond_to? :bcms_sitemap_last_update
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
context 'submitting sitemap' do
|
21
|
+
setup do
|
22
|
+
@search_engine = create_search_engine
|
23
|
+
@submitter = Cms::SitemapSubmitter.new(@search_engine)
|
24
|
+
end
|
25
|
+
should 'return response code on success' do
|
26
|
+
@submitter.instance_variable_get(:@connection).stubs(:get).returns('')
|
27
|
+
resp = @submitter.submit
|
28
|
+
assert_equal 200, resp
|
29
|
+
end
|
30
|
+
should 'return response code on failure' do
|
31
|
+
@submitter.instance_variable_get(:@connection).expects(:get).raises(ActiveResource::ResourceNotFound, 404)
|
32
|
+
resp = @submitter.submit
|
33
|
+
assert_equal 404, resp
|
34
|
+
end
|
35
|
+
should 'generate correct parameters' do
|
36
|
+
assert_equal "http%3A%2F%2Flocalhost%3A3000%2Fsitemaps.xml", @submitter.parameters
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
context 'run' do
|
41
|
+
setup do
|
42
|
+
@search_engine = create_search_engine
|
43
|
+
Cms::SitemapSubmitter.stubs(:submit).returns(200)
|
44
|
+
end
|
45
|
+
|
46
|
+
should 'not submit if nothing updated' do
|
47
|
+
Cms::SitemapSubmitter.run
|
48
|
+
assert_nil @search_engine.reload.submitted_at
|
49
|
+
end
|
50
|
+
|
51
|
+
context 'based on updated' do
|
52
|
+
setup do
|
53
|
+
@search_engine.update_attributes(:submitted_at => Date.yesterday)
|
54
|
+
@last_time = @search_engine.submitted_at
|
55
|
+
end
|
56
|
+
should 'submit if never submitted' do
|
57
|
+
Cms::SitemapSubmitter.run
|
58
|
+
assert_not_nil @search_engine.reload.submitted_at
|
59
|
+
end
|
60
|
+
context 'pages' do
|
61
|
+
setup do
|
62
|
+
@page = create_page
|
63
|
+
end
|
64
|
+
should 'submit if something is updated since last time' do
|
65
|
+
Cms::SitemapSubmitter.run
|
66
|
+
assert_not_equal @last_time, @search_engine.reload.submitted_at
|
67
|
+
end
|
68
|
+
should 'update last_status' do
|
69
|
+
Cms::SitemapSubmitter.run
|
70
|
+
assert_equal 200, @search_engine.reload.last_status
|
71
|
+
end
|
72
|
+
end
|
73
|
+
context 'articles' do
|
74
|
+
setup do
|
75
|
+
@news = create_news_article
|
76
|
+
end
|
77
|
+
should 'submit if something is updated since last time' do
|
78
|
+
Cms::SitemapSubmitter.run
|
79
|
+
assert_not_equal @last_time, @search_engine.reload.submitted_at
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
end
|