blacklight-sitemap 0.0.1

data/Gemfile

@@ -0,0 +1,15 @@
+source "http://rubygems.org"
+# Add dependencies required to use your gem here.
+# Example:
+#   gem "activesupport", ">= 2.3.5"
+
+gem 'nokogiri'
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "rspec", "~> 2.1.0"
+  gem "bundler", "~> 1.0.0"
+  gem "jeweler", "~> 1.5.1"
+  gem "rcov", ">= 0"
+end

data/Gemfile.lock

@@ -0,0 +1,30 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    diff-lcs (1.1.2)
+    git (1.2.5)
+    jeweler (1.5.1)
+      bundler (~> 1.0.0)
+      git (>= 1.2.5)
+      rake
+    nokogiri (1.4.4)
+    rake (0.8.7)
+    rcov (0.9.9)
+    rspec (2.1.0)
+      rspec-core (~> 2.1.0)
+      rspec-expectations (~> 2.1.0)
+      rspec-mocks (~> 2.1.0)
+    rspec-core (2.1.0)
+    rspec-expectations (2.1.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.1.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0.0)
+  jeweler (~> 1.5.1)
+  nokogiri
+  rcov
+  rspec (~> 2.1.0)

data/LICENSE.txt

@@ -0,0 +1,21 @@
+Copyright (c) 2010 North Carolina State University
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

data/README.rdoc

@@ -0,0 +1,41 @@
+= blacklight-sitemap
+
+Rake task for creating a sitemap from a Blacklight Solr index.
+
+== Installation
+
+In config/environment.rb add:
+    config.gem 'blacklight-sitemap'
+
+Install the gem:
+    rake gems:install
+or
+    gem install blacklight-sitemap
+
+Run the generator:
+    script/generate blacklight_sitemap
+
+Open your Rakefile and edit the BlacklightSitemapTask to your liking using the
+provided documentation.
+
+Run the rake task:
+    rake blacklight:sitemap
+
+Remove create sitemap files:
+    rake blacklight:sitemap:clobber
+
+
+== TODO
+
+* Determine the lastmodified date for a particular sub-sitemap. For large indexes, allowing search engines to only download the sitemaps which have changed will be important. Can we always sort by timestamp?
+* How to more fully test the gem independent from a Rails application with Blacklight installed?
+
+== Author
+
+Jason Ronallo
+
+== Copyright
+
+Copyright (c) 2010 North Carolina State University. See LICENSE.txt for
+further details.
+

data/Rakefile

@@ -0,0 +1,51 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "blacklight-sitemap"
+  gem.homepage = "http://github.com/jronallo/blacklight-sitemap"
+  gem.license = "MIT"
+  gem.summary = %Q{Blacklight sitemap rake task.}
+  gem.description = %Q{Blacklight sitemap rake task.}
+  gem.email = "jronallo@gmail.com"
+  gem.authors = ["Jason Ronallo"]
+  # Include your dependencies below. Runtime dependencies are required when using your gem,
+  # and development dependencies are only needed for development (ie running rake tasks, tests, etc)
+  #  gem.add_runtime_dependency 'jabber4r', '> 0.1'
+  #  gem.add_development_dependency 'rspec', '> 1.2.3'
+  gem.files = FileList["[A-Z]*", "{generators,lib}/**/*"]
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+
+RSpec::Core::RakeTask.new(:rcov) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "blacklight-sitemap #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/generators/blacklight_sitemap/blacklight_sitemap_generator.rb

@@ -0,0 +1,53 @@
+class BlacklightSitemapGenerator < Rails::Generator::Base
+  def initialize(*runtime_args)
+    super
+  end
+
+  def manifest
+    record do |m|
+      sitemap_task = <<EOF
+require 'blacklight-sitemap'
+Rake::BlacklightSitemapTask.new do |sm|
+  # below are configuration options with their default values shown.
+
+  # FIXME: you'll definitely want to change the url value
+  # sm.url = 'http://localhost:3000'
+
+  # base filename given to generated sitemap files
+  # sm.base_filename = 'blacklight'
+
+  # Is the gzip commandline tool available? Then why not gzip up your sitemaps to
+  # save bandwidth?
+  # sm.gzip = false
+
+  # for changefreq see http://sitemaps.org/protocol.php#changefreqdef
+  # valid values are: always, hourly, daily, weekly, monthly, yearly, never
+  # sm.changefreq = nil # nil won't display a changefreq element
+
+  # sitemaps can contain up to 50000 locations, but also must not be more than
+  # 10 MB in size. Using the max value you can control the size of your files.
+  # sm.max = 50000
+
+  # Solr field used to retrieve from a document the value for the lastmod element for a url
+  # sm.lastmod_field = 'timestamp'
+
+  # Solr field used to retrieve from a document the value for the priority element for a url
+  # sm.priority_field = nil
+end
+EOF
+      rakefile = File.read('Rakefile')
+      if rakefile.scan('Rake::BlacklightSitemapTask.new').empty?
+        rakefile << "\n" << sitemap_task
+        File.open('Rakefile', 'w'){|f| f.puts rakefile}
+      end
+    end
+  end
+
+  protected
+
+  def banner
+    %{Usage: #{$0} #{spec.name}\nCopies OpenURL.js public/javascripts/}
+  end
+
+end
+

data/lib/blacklight-sitemap.rb

@@ -0,0 +1,122 @@
+#!/usr/bin/env ruby
+require 'nokogiri'
+require 'rake'
+require 'fileutils'
+
+module Rake
+  class BlacklightSitemapTask
+    # base url used for all locations
+    attr_accessor :url
+
+    # base filename to use for sitemap in case these will be moved to a location
+    # that hosts other sitemaps so these sitemaps do not overwrite others
+    attr_accessor :base_filename
+
+    # should the files be gzipped? requires the commandline tool gzip
+    attr_accessor :gzip
+
+    # value for changefreq for each page listed
+    attr_accessor :changefreq
+
+    # the most resources which should be listed within a single sitemap
+    # defaults to 50,000
+    attr_accessor :max
+
+    # Solr field that contains a date to create a lastmod date for the page.
+    # Currently must be a string as in W3C Datetime format or YYYY-MM-DD
+    attr_accessor :lastmod_field
+
+    # Solr field to use to provide a priority for this resource
+    attr_accessor :priority_field
+
+
+    def initialize
+      @url = 'http://localhost:3000'
+      @base_filename = 'blacklight'
+      @gzip = false
+      @changefreq = nil
+      @max = 50000 #default value for max number of locs per sitemap file
+      @lastmod_field = 'timestamp'
+      @priority_field = nil
+      yield self if block_given?
+      define
+    end
+
+    def define
+      namespace :blacklight do
+        desc 'clobber then create sitemap files for blacklight'
+        task :sitemap => ['sitemap:clobber', 'sitemap:create']
+
+        namespace :sitemap do
+
+          desc 'create a sitemap for blacklight'
+          task :create => :environment do
+            puts 'Creating a sitemap...'
+            fl = ['id', @lastmod_field, @priority_field].compact.join(',')
+            base_solr_parameters = {:qt => 'standard', :q => 'id:[* TO *]', :fl => fl}
+            number_of_resources = Blacklight.solr.find(base_solr_parameters.merge(:rows => 1))['response']['numFound']
+            puts 'Number of resources: ' + number_of_resources.to_s
+            batches = (number_of_resources / @max.to_f).ceil
+            puts 'Total sitemap to create: ' + batches.to_s
+            master_sitemap = ''
+
+            batches.times do |batch_number|
+              current_page = batch_number + 1
+              start = batch_number * @max
+              puts 'Processing batch # ' + current_page.to_s
+              response = Blacklight.solr.find(base_solr_parameters.merge(:rows => @max, :start => start))['response']
+              sitemap_builder = Nokogiri::XML::Builder.new do |xml|
+                xml.urlset "xmlns" => "http://www.sitemaps.org/schemas/sitemap/0.9" do
+                  response['docs'].each do |doc|
+                    xml.url do
+                      # FIXME through config
+                      xml.loc File.join(@url.to_s, doc['id'])
+                      xml.lastmod doc[@lastmod_field].to_s if @lastmod_field and doc[@lastmod_field]
+                      xml.priority doc[@priority_field] if @priority_field and doc[@priority_field]
+                      xml.changefreq @changefreq if @changefreq
+                    end
+                  end
+                end
+              end
+              sitemap_filename = File.join(RAILS_ROOT, 'public', @base_filename + '-sitemap' + batch_number.to_s + '.xml')
+              File.open(sitemap_filename, 'w') do |fh|
+                fh.puts sitemap_builder.to_xml
+              end
+              if @gzip
+                `gzip #{sitemap_filename}`
+              end
+            end
+            puts 'Creating sitemap index...'
+            lastmod = DateTime.now.utc.strftime("%Y-%m-%dT%H:%M:%S+00:00")
+            sitemap_index_builder = Nokogiri::XML::Builder.new do |xml|
+              xml.sitemapindex 'xmlns' => 'http://www.sitemaps.org/schemas/sitemap/0.9' do
+                batches.times do |batch|
+                  sitemap_filename = File.join(@url.to_s, @base_filename + '-sitemap' + batch.to_s + '.xml')
+                  sitemap_filename << '.gz' if @gzip
+                  xml.sitemap{
+                    xml.loc sitemap_filename
+                    xml.lastmod lastmod
+                  }
+                end
+              end
+            end #sitemap_index_builder
+            File.open(File.join(RAILS_ROOT, 'public', @base_filename + '-sitemap.xml'), 'w') do |fh|
+              fh.puts sitemap_index_builder.to_xml
+            end
+            puts 'Done.'
+          end # task :sitemap
+
+          desc 'clobber sitemap files'
+          task :clobber do
+            puts "Deleting all sitemap files..."
+            Dir.glob(File.join(RAILS_ROOT, 'public', @base_filename + '-sitemap*')).each do |sitemap|
+              FileUtils.rm(sitemap)
+            end
+          end
+
+        end # namespace :sitemap
+      end # namespace :blacklight
+    end # define
+  end # BlacklightSitemapTask
+end # Rake
+

data/spec/blacklight-sitemap_spec.rb

@@ -0,0 +1,93 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "BlacklightSitemap" do
+  before(:all) do
+    @default_task = Rake::BlacklightSitemapTask.new
+  end
+
+  it 'should create a default task which clobbers then creates' do
+    Rake::BlacklightSitemapTask.new
+    Rake::Task['blacklight:sitemap'].should be_a_kind_of Rake::Task
+  end
+
+  it "should create the sitemap creation task" do
+    Rake::BlacklightSitemapTask.new
+    Rake::Task['blacklight:sitemap:create'].should be_a_kind_of Rake::Task
+  end
+
+  it "should be able to have the url attribute" do
+    task = Rake::BlacklightSitemapTask.new do |sm|
+      sm.url = 'http://example.com'
+    end
+    task.url.should eq('http://example.com')
+  end
+
+  it 'should have a base_filename attribute' do
+    @default_task.base_filename.should eq('blacklight')
+  end
+
+  it 'should allow for changing the base_filename attribute' do
+    task = Rake::BlacklightSitemapTask.new do |sm|
+      sm.base_filename = 'bl'
+    end
+    task.base_filename.should eq('bl')
+  end
+
+  it 'should store a value for whether to gzip or not' do
+    task = Rake::BlacklightSitemapTask.new do |sm|
+      sm.gzip = true
+    end
+    task.gzip.should be_true
+  end
+
+  it 'should have a default nil value for changefreq' do
+    @default_task.changefreq.should be_nil
+  end
+
+  it 'should allow for changing the changefreq value' do
+    task = Rake::BlacklightSitemapTask.new do |sm|
+      sm.changefreq = 'never'
+    end
+    task.changefreq.should eq('never')
+  end
+
+  it 'should set the ceiling to 50,000 as a default' do
+    @default_task.max.should == 50000
+  end
+
+  it 'should allow the ceiling to be set to a different value' do
+    task = Rake::BlacklightSitemapTask.new do |sm|
+      sm.max = 50
+    end
+    task.max.should == 50
+  end
+
+  it 'should have a default value for the lastmod_field' do
+    @default_task.lastmod_field.should eq('timestamp')
+  end
+
+  it 'should allow a new value for lastmod_field' do
+    task = Rake::BlacklightSitemapTask.new do |sm|
+      sm.lastmod_field = 'date_created'
+    end
+    task.lastmod_field.should eq('date_created')
+  end
+
+  it 'should have a default value for the priority_field' do
+    @default_task.priority_field.should be_nil
+  end
+
+  it 'should allow a new value for priority_field' do
+    task = Rake::BlacklightSitemapTask.new do |sm|
+      sm.priority_field = 'priority'
+    end
+    task.priority_field.should eq('priority')
+  end
+
+  it 'should create the sitemap clobber task' do
+    Rake::BlacklightSitemapTask.new
+    Rake::Task['blacklight:sitemap:clobber'].should be_a_kind_of Rake::Task
+  end
+
+end
+

data/spec/spec_helper.rb

@@ -0,0 +1,12 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'rspec'
+require 'blacklight-sitemap'
+
+# Requires supporting files with custom matchers and macros, etc,
+# in ./support/ and its subdirectories.
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
+
+RSpec.configure do |config|
+  
+end

metadata

@@ -0,0 +1,153 @@
+--- !ruby/object:Gem::Specification 
+name: blacklight-sitemap
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Jason Ronallo
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-12-02 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id001
+  prerelease: false
+  type: :runtime
+  name: nokogiri
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 11
+        segments: 
+        - 2
+        - 1
+        - 0
+        version: 2.1.0
+  requirement: *id002
+  prerelease: false
+  type: :development
+  name: rspec
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 1
+        - 0
+        - 0
+        version: 1.0.0
+  requirement: *id003
+  prerelease: false
+  type: :development
+  name: bundler
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 1
+        segments: 
+        - 1
+        - 5
+        - 1
+        version: 1.5.1
+  requirement: *id004
+  prerelease: false
+  type: :development
+  name: jeweler
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id005
+  prerelease: false
+  type: :development
+  name: rcov
+description: Blacklight sitemap rake task.
+email: jronallo@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE.txt
+- README.rdoc
+files: 
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.rdoc
+- Rakefile
+- VERSION
+- generators/blacklight_sitemap/blacklight_sitemap_generator.rb
+- lib/blacklight-sitemap.rb
+- spec/blacklight-sitemap_spec.rb
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://github.com/jronallo/blacklight-sitemap
+licenses: 
+- MIT
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Blacklight sitemap rake task.
+test_files: 
+- spec/blacklight-sitemap_spec.rb
+- spec/spec_helper.rb