blacklight_dynamic_sitemap 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 1d40182b152ba0d7cb73a9bbc6f01afc84b860baad5f7e8b66357e5beb97ae75
+  data.tar.gz: 1cf4281e052c836ce59aee3ef135dfad28c8ed82314ad6f3a4e5111251aa2032
+SHA512:
+  metadata.gz: de245a04d2073183f62631d082875fabffc88dbd766ed0f7441eff3beb9dbca1126f2748cbe4f97c8bacab4fb3e0fa9a4121803c20bb5a9322877bb2835ddc46
+  data.tar.gz: b4ca278cd5530412c70596c54fe9650f7e3887c292eb65b1489dad37dc750d1e018145dad080e5a2fabfcdd27b121796b63cda84b941ce133cec1aaacfdba316

data/.gitignore

@@ -0,0 +1,15 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+
+Gemfile.lock
+
+.internal_test_app

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.solr_wrapper

@@ -0,0 +1,5 @@
+# Place any default configuration for solr_wrapper here
+# port: 8983
+collection:
+  dir: solr/conf/
+  name: blacklight-core

data/.travis.yml

@@ -0,0 +1,8 @@
+---
+language: ruby
+cache: bundler
+rvm:
+  - 2.6.3
+notifications:
+  email: false
+jdk: openjdk11

data/Gemfile

@@ -0,0 +1,45 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in blacklight_dynamic_sitemap.gemspec
+gemspec
+
+gem 'rake', '~> 12.0'
+gem 'rspec', '~> 3.0'
+# BEGIN ENGINE_CART BLOCK
+# engine_cart: 2.3.0
+# engine_cart stanza: 0.10.0
+# the below comes from engine_cart, a gem used to test this Rails engine gem in the context of a Rails app.
+file = File.expand_path('Gemfile', ENV['ENGINE_CART_DESTINATION'] || ENV['RAILS_ROOT'] || File.expand_path('.internal_test_app', File.dirname(__FILE__)))
+if File.exist?(file)
+  begin
+    eval_gemfile file
+  rescue Bundler::GemfileError => e
+    Bundler.ui.warn '[EngineCart] Skipping Rails application dependencies:'
+    Bundler.ui.warn e.message
+  end
+else
+  Bundler.ui.warn "[EngineCart] Unable to find test application dependencies in #{file}, using placeholder dependencies"
+
+  if ENV['RAILS_VERSION']
+    if ENV['RAILS_VERSION'] == 'edge'
+      gem 'rails', github: 'rails/rails'
+      ENV['ENGINE_CART_RAILS_OPTIONS'] = '--edge --skip-turbolinks'
+    else
+      gem 'rails', ENV['RAILS_VERSION']
+    end
+  end
+
+  case ENV['RAILS_VERSION']
+  when /^4.2/
+    gem 'responders', '~> 2.0'
+    gem 'sass-rails', '>= 5.0'
+    gem 'coffee-rails', '~> 4.1.0'
+  when /^4.[01]/
+    gem 'sass-rails', '< 5.0'
+  end
+end
+# END ENGINE_CART BLOCK
+
+unless File.exist?(file)
+  eval_gemfile File.expand_path('spec/test_app_templates/Gemfile.extra', File.dirname(__FILE__))
+end

data/LICENSE

@@ -0,0 +1,13 @@
+© 2020 The Board of Trustees of the Leland Stanford Junior University.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,44 @@
+# BlacklightDynamicSitemap
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/blacklight_dynamic_sitemap`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'blacklight_dynamic_sitemap'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install blacklight_dynamic_sitemap
+
+Run the install generator
+
+    $ bundle exec rails generate blacklight_dynamic_sitemap:install
+
+## Usage
+
+### Solr configuration
+This feature relies on a unique hexadecimal hash field in your solr response for each document.  You can have solr automatically do this for you using the `SignatureUpdateProcessorFactory`.
+
+https://lucene.apache.org/solr/guide/8_4/update-request-processors.html
+
+See the `updateRequestProcessorChain` used in this project's `solrconfig.xml` for an example.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/blacklight_dynamic_sitemap.

data/Rakefile

@@ -0,0 +1,13 @@
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+Bundler::GemHelper.install_tasks
+
+load 'tasks/blacklight_dynamic_sitemap.rake'
+
+require 'engine_cart/rake_task'
+
+task default: [:ci]

data/app/controllers/blacklight_dynamic_sitemap/sitemap_controller.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module BlacklightDynamicSitemap
+  ##
+  # Controller to handle the on-the-fly sitemap generation
+  class SitemapController < ApplicationController
+    def show
+      @sitemap_entries = BlacklightDynamicSitemap::Sitemap.new.get(id)
+    end
+
+    def index
+      @sitemaps = BlacklightDynamicSitemap::Sitemap.new.list
+    end
+
+    private
+
+    def id
+      params.require(:id)
+    end
+  end
+end

data/app/models/blacklight_dynamic_sitemap/sitemap.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module BlacklightDynamicSitemap
+  ##
+  #
+  class Sitemap
+    delegate :hashed_id_field, :unique_id_field, :last_modified_field, to: :engine_config
+
+    def get(id)
+      index_connection.select(
+        params: {
+          q: "{!prefix f=#{hashed_id_field} v=#{id}}",
+          fl: [unique_id_field, last_modified_field].join(','),
+          rows: 2_000_000, # Ensure that we do not page this result
+          facet: false
+        }
+      ).dig('response', 'docs')
+    end
+
+    def list
+      access_list
+    end
+
+    private
+
+    def index_connection
+      @index_connection ||= Blacklight.default_index.connection
+    end
+
+    def engine_config
+      BlacklightDynamicSitemap::Engine.config
+    end
+
+    def max_documents
+      key = 'blacklight_dynamic_sitemap.index_max_docs'
+      expiration = BlacklightDynamicSitemap::Engine.config.max_documents_expiration
+      Rails.cache.fetch(key, expires_in: expiration) do
+        Blacklight.default_index.connection.select(
+          params: { q: '*:*', rows: 0, facet: false }
+        )['response']['numFound']
+      end
+    end
+
+    def average_chunk
+      [engine_config.minimum_average_chunk, max_documents].min # Sufficiently less than 50,000 max per sitemap
+    end
+
+    ##
+    # Exponent used to calculate the needed number of prefix spaces to query
+    # that will effectively chunk the entire number of documents. 16 is the
+    # number of characters in hex space (0-9, a-f)
+    # Example: 16**1 = 16, 16**2 = 256, 16**3 = 4096
+    # x = b**y
+    # y = logb(x)
+    # y = logb(x) = ln(x) / ln(b)
+    def exponent
+      @exponent ||= [
+        (Math.log(max_documents / average_chunk) / Math.log(16)).ceil,
+        1
+      ].max
+    end
+
+    ##
+    # Expand the number of documents used off of calculated exponent to create
+    # list of sitemaps to access in hex space (0-9, a-f)
+    # Example: (exponent as 4)
+    # ["0000", "0001", "0002", "0003"..."af74", "af75", "af76", "af77"..."fffc", "fffd", "fffe", "ffff"]
+    def access_list
+      (0...(16**exponent))
+        .to_a
+        .map { |v| v.to_s(16).rjust(exponent, '0') }
+    end
+  end
+end

data/app/views/blacklight_dynamic_sitemap/sitemap/index.xml.builder

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+xml.instruct! :xml, version: '1.0', encoding: 'UTF-8'
+xml.sitemapindex(
+  'xmlns:xsi' => 'http://www.w3.org/2001/XMLSchema-instance',
+  'xsi:schemaLocation' => 'http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/siteindex.xsd',
+  'xmlns' => 'http://www.sitemaps.org/schemas/sitemap/0.9'
+) do
+  @sitemaps.each do |id|
+    xml.sitemap do
+      xml.loc(sitemap_url(id, format: :xml))
+    end
+  end
+end

data/app/views/blacklight_dynamic_sitemap/sitemap/show.xml.builder

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+xml.instruct! :xml, version: '1.0', encoding: 'UTF-8'
+xml.urlset(
+  'xmlns:xsi' => 'http://www.w3.org/2001/XMLSchema-instance',
+  'xsi:schemaLocation' => 'http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd',
+  'xmlns' => 'http://www.sitemaps.org/schemas/sitemap/0.9'
+) do
+  @sitemap_entries.each do |doc|
+    xml.url do
+      xml.loc(main_app.solr_document_url(doc[BlacklightDynamicSitemap::Engine.config.unique_id_field]))
+      xml.lastmod(doc[BlacklightDynamicSitemap::Engine.config.last_modified_field])
+    end
+  end
+end

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'blacklight_dynamic_sitemap'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require 'pry'
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/blacklight_dynamic_sitemap.gemspec

@@ -0,0 +1,34 @@
+require_relative 'lib/blacklight_dynamic_sitemap/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'blacklight_dynamic_sitemap'
+  spec.version       = BlacklightDynamicSitemap::VERSION
+  spec.authors       = ['Jack Reed']
+  spec.email         = ['phillipjreed@gmail.com']
+
+  spec.summary       = 'Dynamic sitemap.xml for BLacklight'
+  spec.description   = 'Dynamic sitemap.xml for BLacklight'
+  spec.homepage      = 'https://github.com/sul-dlss/blacklight_dynamic_sitemap'
+  spec.license       = 'Apache-2.0'
+  spec.required_ruby_version = Gem::Requirement.new('>= 2.3.0')
+
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = spec.homepage
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'rails', '~> 5.0'
+  spec.add_dependency 'blacklight', '> 6.0'
+
+  spec.add_development_dependency 'capybara'
+  spec.add_development_dependency 'engine_cart', '~> 2.0'
+  spec.add_development_dependency 'rspec-rails'
+  spec.add_development_dependency 'solr_wrapper'
+end

data/config/routes.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+BlacklightDynamicSitemap::Engine.routes.draw do
+  resources :sitemap, controller: 'sitemap', only: %i[index show], defaults: { format: :xml }
+end

data/lib/blacklight_dynamic_sitemap.rb

@@ -0,0 +1,8 @@
+require 'blacklight_dynamic_sitemap/version'
+require 'blacklight'
+
+module BlacklightDynamicSitemap
+  require 'blacklight_dynamic_sitemap/engine'
+  class Error < StandardError; end
+  # Your code goes here...
+end

data/lib/blacklight_dynamic_sitemap/engine.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module BlacklightDynamicSitemap
+  class Engine < ::Rails::Engine
+    isolate_namespace BlacklightDynamicSitemap
+    config.max_documents_expiration = 1.day
+    config.minimum_average_chunk = 10_000
+    config.hashed_id_field = 'hashed_id_ssi'
+    config.unique_id_field = 'id'
+    config.last_modified_field = 'timestamp'
+  end
+end

data/lib/blacklight_dynamic_sitemap/version.rb

@@ -0,0 +1,3 @@
+module BlacklightDynamicSitemap
+  VERSION = '0.1.0'
+end

data/lib/generators/blacklight_dynamic_sitemap/install_generator.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'rails/generators'
+
+module BlacklightDynamicSitemap
+  class Install < Rails::Generators::Base
+    def add_routes
+      inject_into_file 'config/routes.rb', after: "mount Blacklight::Engine => '/'" do
+        "\n  mount BlacklightDynamicSitemap::Engine => '/'\n"
+      end
+    end
+  end
+end

data/solr/conf/_rest_managed.json

@@ -0,0 +1,3 @@
+{
+  "initArgs":{},
+  "managedList":[]}

data/solr/conf/admin-extra.html

@@ -0,0 +1,31 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- The content of this page will be statically included into the top
+of the admin page.  Uncomment this as an example to see there the content
+will show up.
+
+<hr>
+<i>This line will appear before the first table</i>
+<tr>
+<td colspan="2">
+This row will be appended to the end of the first table
+</td>
+</tr>
+<hr>
+
+-->