kithe 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 423d120379d13836cc034d1884b813c8f6a2505c77a1e9edcdd8b4fbb33a0025
-  data.tar.gz: d9bfc7bc5f60a86f2e2e6726440de4b319cfa64526313a999af269340ec0cf1d
+  metadata.gz: d81e5ad94edad9885f8b51d4adc04c64093faa6b105a4e0c9dd5caa683b116cd
+  data.tar.gz: 21e0f6ff7aa02a592216c17e3cbda31341ffcec92835e04141d0fb862787de84
 SHA512:
-  metadata.gz: 3696eebb93e5975e8336bf79018f3c1c2fd5e49015813442e5586794ad12193a1f843a43ae4a4d238eff906135434547473c49e327c6d03522f92766f6759c5b
-  data.tar.gz: a7761da6c072f0bdbc48468227177244ab15a8e46d392ae930e8734e555134c237f4aecb928aef6ab669e430752bd838c803844b2fe30a79a44b7e8827121ddf
+  metadata.gz: bf07407aa6e23d2460f58e5d636333eb3fa72977d210d993a78e616813a2f47fec1f426cd22e4ad58f0d648a435ae8be77fa003a7c530de9d11a30d768323fd3
+  data.tar.gz: ce7f84ec2833f5fa5ed4aa07b7ffc7cbbd187d3aa27adeec2282ccf380bf0d137e73ac3d64900b9b6c6ae65f1ecbc268711d193b767e07e3a348f3f94eac9133

data/README.md

@@ -44,6 +44,8 @@ Some guide documentation is available to explain each of kithe's major functiona
   * Not coupled to any other kithe components, could be used independently, hypothetically on any ActiveRecord model.
   * Written after review of "prior art"  in [sunspot](https://github.com/sunspot/sunspot) and [searchkick](https://github.com/ankane/searchkick) (which both used AR callback-based indexing), and others.
 
+* A [recommended approach for using Blacklight](./guides/blacklight_approach.md) with search result view templates based on actual ActiveRecord models. It is totally optional to use Blacklight at all with kithe, or to use this approach if you do.
+
 ### Also
 
 * [Kithe::Parameters](./app/models/kithe/parameters.rb) provides some shortcuts around Rails "strong params" for attr_json serialized attributes.

data/app/indexing/kithe/indexable.rb

@@ -9,7 +9,7 @@ module Kithe
   # For a complete overview, see the [Guide Documentation](../../../guides/solr_indexing.md)
   #
   # The Solr instance to send updates to is global configuration:
-  #     Kithe::Indexable.settings.solr_url = "http://localhost:8983/solr/collection_name"
+  #     Kithe.indexable_settings.solr_url = "http://localhost:8983/solr/collection_name"
   #
   # To configure how a model is mapped to a Solr document, you create a `Kithe::Indexer` sub-class, which
   # can use our obj_extract method, as well as any other traject indexer code.
@@ -52,91 +52,6 @@ module Kithe
   module Indexable
     extend ActiveSupport::Concern
 
-    class IndexableSettings
-      attr_accessor :solr_url, :writer_class_name, :writer_settings, :model_name_solr_field, :disable_callbacks
-      def initialize(solr_url:, writer_class_name:, writer_settings:, model_name_solr_field:, disable_callbacks: false)
-        @solr_url = solr_url
-        @writer_class_name = writer_class_name
-        @writer_settings = writer_settings
-        @model_name_solr_field = model_name_solr_field
-      end
-
-      # Use configured solr_url, and merge together with configured
-      # writer_settings
-      def writer_settings
-        if solr_url
-          { "solr.url" => solr_url }.merge(@writer_settings)
-        else
-          @writer_settings
-        end
-      end
-
-      # Turn writer_class_name into an actual Class object.
-      def writer_class
-        writer_class_name.constantize
-      end
-
-      # Instantiate a new writer based on `writer_class_name` and `writer_settings`
-      def writer_instance!(additional_settings = {})
-        writer_class.new(writer_settings.merge(additional_settings))
-      end
-    end
-
-    # Global Kithe::Indexable settings, actually a Kithe::Indexable::Settings
-    # object, but you will generally use it as a simple value object with getters
-    # and setters.
-    #
-    # * solr_url: Where to send to Solr when indexing, the base url
-    #
-    #     Kithe::Indexable.settings.solr_url = "http://localhost:8983/solr/collection_name"
-    #
-    # * model_name_solr_field: If you'd like a custom solr field to store model class name in.
-    #
-    #     Kithe::Indexable.settings.model_name_solr_field = "my_model_name_field"
-    #
-    # * writer_settings: Settings to be passed to the Traject writer, by default a
-    #   Traject::SolrJsonWriter. To maintain the default settings, best to merge
-    #   your new ones into defaults.
-    #
-    #       Kithe::Indexable.settings.writer_settings.merge!(
-    #         # by default we send a softCommit on every update, maybe you
-    #         # want not to?
-    #         "solr_writer.solr_update_args" => {}
-    #         # extra long timeout?
-    #         "solr_writer.http_timeout" => 100
-    #       )
-    #
-    # * writer_class_name: By default Traject::SolrJsonWriter, but maybe
-    #   you want to set to some other Traject::Writer. The writer Kithe::Indexable
-    #   will send index add/remove requests to.
-    #
-    #       Kithe::Indexable.settings.writer_class_name = "Traject::SomeOtherWriter"
-    #
-    # * disable_callbacks: set to true to globally disable automatic after_commit
-    mattr_accessor :settings do
-      # set up default settings
-      IndexableSettings.new(
-        solr_url: "http://localhost:8983/solr/default",
-        model_name_solr_field: "model_name_ssi",
-        writer_class_name: "Traject::SolrJsonWriter",
-        writer_settings: {
-          # as default we tell the solrjsonwriter to use no threads,
-          # no batching. softCommit on every update. Least surprising
-          # default configuration.
-          "solr_writer.thread_pool" => 0,
-          "solr_writer.batch_size" => 1,
-          "solr_writer.solr_update_args" => { softCommit: true },
-          "solr_writer.http_timeout" => 3,
-          "logger" => Rails.logger,
-
-          # MAYBE? no skippable exceptions please
-          # "solr_writer.skippable_exceptions" => []
-        },
-        disable_callbacks: false
-      )
-    end
-
-
     # Set some indexing parameters for the block yielded. For instance, to batch updates:
     #
     #     Kithe::Indexable.index_with(batching: true)
@@ -163,12 +78,12 @@ module Kithe
 
     # Are automatic after_commit callbacks currently enabled? Will check a number
     # of things to see, as we have a number of places these can be turned on/off.
-    # * Globally in `Kithe::Indexable.settings.disable_callback`
+    # * Globally in `Kithe.indexable_settings.disable_callback`
     # * On class or instance using class_attribute `kithe_indexable_auto_callbacks`
     # * If no kithe_indexable_mapper is configured on record, then no callbacks.
     # * Using thread-current settings usually set by .index_with
     def self.auto_callbacks?(model)
-      !Kithe::Indexable.settings.disable_callbacks &&
+      !Kithe.indexable_settings.disable_callbacks &&
         model.kithe_indexable_auto_callbacks &&
         model.kithe_indexable_mapper &&
         !ThreadSettings.current.disabled_callbacks?

data/app/indexing/kithe/indexable/record_index_updater.rb

@@ -68,7 +68,7 @@ module Kithe
       # Could be an explicit writer passed into #initialize, or a current thread-settings
       # writer, or a new writer created from global settings.
       def writer
-        @writer ||= ThreadSettings.current.writer  || Kithe::Indexable.settings.writer_instance!
+        @writer ||= ThreadSettings.current.writer  || Kithe.indexable_settings.writer_instance!
       end
 
       # Is this record supposed to be represented in the solr index?

data/app/indexing/kithe/indexable/thread_settings.rb

@@ -81,7 +81,7 @@ module Kithe
         @writer ||= begin
           if @batching
             @local_writer = true
-            Kithe::Indexable.settings.writer_instance!("solr_writer.batch_size" => 100)
+            Kithe.indexable_settings.writer_instance!("solr_writer.batch_size" => 100)
           end
         end
       end

data/app/indexing/kithe/indexer.rb

@@ -18,9 +18,14 @@ module Kithe
   # * A Kithe::Indexer will automatically index the source record #id to Solr object
   #   #id, and the source record class name to Solr field `model_name_ssi`. (That uses
   #   Blacklight conventions for dynamic field names, if you'd like to change the field name
-  #   used, set `Kithe::Indexable.settings.model_name_solr_field=`)
+  #   used, set `Kithe.indexable_settings.model_name_solr_field=`)
   #
   # *  ID and model_name are set, so the AR object can be easily fetched later from Solr results.
+  #   * You can customize what Solr field the model_name is sent to with
+  #     `Kithe.indexable_settings.model_name_solr_field=`, by default `model_name_ssi`, using
+  #     a blacklight dynamic field template `*_ssi`.
+  #   * You can customize what ActiveRecord model property is sent to Solr `id` field with
+  #     `Kithe.indexable_settings.solr_id_value_attribute=`, by default the AR pk in model#id.
   #
   # Note that there are no built-in facilities for automatically sending every field of your model
   # to Solr, round-trippable or not. The expected usage pattern is sending to Solr only
@@ -43,9 +48,8 @@ module Kithe
     #
     # TODO We might not actually want to do these automatically, or allow it to be disabled?
     configure do
-      # hard-coded id -> id for now. id is a UUID. Can be made configurable?
-      to_field "id", obj_extract("id")
-      to_field Kithe::Indexable.settings.model_name_solr_field, obj_extract("class", "name")
+      to_field "id", obj_extract(Kithe.indexable_settings.solr_id_value_attribute)
+      to_field Kithe.indexable_settings.model_name_solr_field, obj_extract("class", "name")
     end
   end
 end

data/app/indexing/kithe/solr_util.rb

@@ -16,7 +16,7 @@ module Kithe
     #        delete(id)
     #     end
     #
-    # It is searching for any Solr object with a `Kithe::Indexable.settings.model_name_solr_field`
+    # It is searching for any Solr object with a `Kithe.indexable_settings.model_name_solr_field`
     # field (default `model_name_ssi`). Then, it takes the ID and makes sure it exists in
     # the database using Kithe::Model. At the moment we are assuming everything is in Kithe::Model,
     # rather than trying to use the `model_name_ssi` to fetch from different tables. Could
@@ -26,10 +26,12 @@ module Kithe
     #
     # A bit hacky implementation, it might be nice to support a progress bar, we
     # don't now.
-    def self.solr_orphan_ids(batch_size: 100, solr_url: Kithe::Indexable.settings.solr_url)
+    def self.solr_orphan_ids(batch_size: 100, solr_url: Kithe.indexable_settings.solr_url)
       return enum_for(:solr_index_orphan_ids) unless block_given?
 
-      model_name_solr_field = Kithe::Indexable.settings.model_name_solr_field
+      model_name_solr_field = Kithe.indexable_settings.model_name_solr_field
+      model_solr_id_attr   = Kithe.indexable_settings.solr_id_value_attribute
+
       solr_page = -1
 
       rsolr = RSolr.connect :url => solr_url
@@ -46,14 +48,14 @@ module Kithe
 
         break if solr_ids.empty?
 
-        (solr_ids - Kithe::Model.where(id: solr_ids).pluck(:id)).each do |orphaned_id|
+        (solr_ids - Kithe::Model.where(model_solr_id_attr => solr_ids).pluck(model_solr_id_attr)).each do |orphaned_id|
           yield orphaned_id
         end
       end
     end
 
     # Finds any Solr objects that have a `model_name_ssi` field
-    # (or `Kithe::Indexable.settings.model_name_solr_field` if non-default), but don't
+    # (or `Kithe.indexable_settings.model_name_solr_field` if non-default), but don't
     # exist in the rdbms, and deletes them from Solr, then issues a commit.
     #
     # Under normal use, you shouldn't have to do this, but can if your Solr index
@@ -65,7 +67,7 @@ module Kithe
     # A bit hacky implementation, it might be nice to have a progress bar, we don't now.
     #
     # Does return an array of any IDs deleted.
-    def self.delete_solr_orphans(batch_size: 100, solr_url: Kithe::Indexable.settings.solr_url)
+    def self.delete_solr_orphans(batch_size: 100, solr_url: Kithe.indexable_settings.solr_url)
       rsolr = RSolr.connect :url => solr_url
       deleted_ids = []
 
@@ -83,10 +85,20 @@ module Kithe
     # using Rsolr. Pretty trivial.
     #
     # Intended for dev/test instances, not really production.
-    def self.delete_all(solr_url: Kithe::Indexable.settings.solr_url)
+    # @param commit :soft, :hard, or false. Default :hard
+    def self.delete_all(solr_url: Kithe.indexable_settings.solr_url, commit: :hard)
       rsolr = RSolr.connect :url => solr_url
-      rsolr.delete_by_query("*:*")
-      rsolr.commit
+
+      # RSolr is a bit under-doc'd, but this SEEMS to work to send a commit
+      # or softCommit instruction with the delete request.
+      params = {}
+      if commit == :hard
+        params[:commit] = true
+      elsif commit == :soft
+        params[:softCommit] = true
+      end
+
+      rsolr.delete_by_query("*:*", params: params)
     end
   end
 end

data/lib/kithe.rb

@@ -1,4 +1,5 @@
 require "kithe/engine"
+require 'kithe/indexable_settings'
 
 module Kithe
   # for ruby-progressbar
@@ -15,4 +16,67 @@ module Kithe
   def self.railtie_namespace
     Kithe::Engine
   end
+
+  # Global Kithe::Indexable settings, actually a Kithe::IndexableSettings
+  # object, but you will generally use it as a simple value object with getters
+  # and setters.
+  #
+  # * solr_url: Where to send to Solr when indexing, the base url
+  #
+  #     Kithe.indexable_settings.solr_url = "http://localhost:8983/solr/collection_name"
+  #
+  # * model_name_solr_field: If you'd like a custom solr field to store model class name in.
+  #
+  #     Kithe.indexable_settings.model_name_solr_field = "my_model_name_field"
+  #
+  # * solr_id_value_attribute: What attribute from your AR models to send to Solr
+  #   `id` uniqueKey field, default the AR `id` pk, you may wish to set to `friendlier_id`.
+  #
+  # * writer_settings: Settings to be passed to the Traject writer, by default a
+  #   Traject::SolrJsonWriter. To maintain the default settings, best to merge
+  #   your new ones into defaults.
+  #
+  #       Kithe.indexable_settings.writer_settings.merge!(
+  #         # by default we send a softCommit on every update, maybe you
+  #         # want not to?
+  #         "solr_writer.solr_update_args" => {}
+  #         # extra long timeout?
+  #         "solr_writer.http_timeout" => 100
+  #       )
+  #
+  # * writer_class_name: By default Traject::SolrJsonWriter, but maybe
+  #   you want to set to some other Traject::Writer. The writer Kithe::Indexable
+  #   will send index add/remove requests to.
+  #
+  #       Kithe.indexable_settings.writer_class_name = "Traject::SomeOtherWriter"
+  #
+  # * disable_callbacks: set to true to globally disable automatic after_commit
+  #
+  #
+  # The settings need to live here not in Kithe::Indexable, to avoid terrible
+  # Rails dev-mode class-reloading weirdnesses. This module is not reloaded.
+  mattr_accessor :indexable_settings do
+  # set up default settings
+  IndexableSettings.new(
+    solr_url: "http://localhost:8983/solr/default",
+    model_name_solr_field: "model_name_ssi",
+    solr_id_value_attribute: "id",
+    writer_class_name: "Traject::SolrJsonWriter",
+    writer_settings: {
+      # as default we tell the solrjsonwriter to use no threads,
+      # no batching. softCommit on every update. Least surprising
+      # default configuration.
+      "solr_writer.thread_pool" => 0,
+      "solr_writer.batch_size" => 1,
+      "solr_writer.solr_update_args" => { softCommit: true },
+      "solr_writer.http_timeout" => 3,
+      "logger" => Rails.logger,
+
+      # MAYBE? no skippable exceptions please
+      # "solr_writer.skippable_exceptions" => []
+    },
+    disable_callbacks: false
+  )
+end
+
 end

data/lib/kithe/blacklight_tools/bulk_loading_search_service.rb

@@ -0,0 +1,38 @@
+require 'kithe/blacklight_tools/search_service_bulk_load'
+
+module Kithe
+  module BlacklightTools
+    # A convenience sub-class of Blacklight::SearchService that
+    # _just_ includes Kithe::BlacklightTools::SearchServiceBulkLoad.
+    #
+    # So if you just need a stock Blacklight::SearchService with this
+    # functionality, in your CatalogController you can conveniently simply:
+    #
+    #     require 'kithe/blacklight_tools/bulk_loading_search_service'
+    #     class CatalogController < ApplicationController
+    #       include Blacklight::Catalog
+    #       # ...
+    #
+    #       self.search_service_class = Kithe::BlacklightTools::BulkLoadingSearchService
+    #
+    #       # ...
+    #     end
+    #
+    # Do NOT sub-class this BulkLoadingSearchService in a local app or gem.
+    # If you need more things in a SearchService, instead make your own
+    # SearchService subclass and
+    # `include Kithe::BlacklightTools::SearchServiceBulkLoad` directly.
+    # This class is simply a convenience for when you need nothing else.
+    #
+    # Kithe devs: Don't add anything to this class beyond
+    # `include Kithe::BlacklightTools::SearchServiceBulkLoad`, so that remains true!
+    #
+    # Note: This is in `./lib` rather than `./app` so it should never get
+    # auto-loaded by the app, as kithe does not require Blacklight and loading
+    # this file without Blacklight would produce an error. Thus the need
+    # for the explicit "require"
+    class BulkLoadingSearchService < Blacklight::SearchService
+      include Kithe::BlacklightTools::SearchServiceBulkLoad
+    end
+  end
+end

data/lib/kithe/blacklight_tools/search_service_bulk_load.rb

@@ -0,0 +1,54 @@
+module Kithe
+  module BlacklightTools
+    # Mix-in module to a Blacklight::SearchService, that will bulk load actual AR
+    # records corresponding to Solr hits, and set them as `model` attribute on each
+    # SolrDocument in the results.
+    #
+    # A very basic rough implementation for basic use cases.
+    #
+    # * Assumes all documents that come back in the Solr results was indexed Kithe::Model, and
+    #   their Solr ID's are the Kithe::Model `id` pk, or from the AR model attribute name
+    #   set in `Kithe.indexable_settings.solr_id_value_attribute`
+    #
+    # * Requires your SolrDocument class to have a `model` attribute, you can just add
+    #   `attr_accessor :model` to your local SolrDocument class BL generated in
+    #   `./app/models/solr_document.rb`. Loaded models will be stored there on your results.
+    #
+    # Just `include` this model in a Blacklight::SearchService subclass. If you need no
+    # additional SearchService customization, but just the standard Blacklight::SearchService
+    # with this functionality, for convenience see the Kithe::BlacklightTools::BulkLoadingSearchServicce
+    #
+    # SORRY: No automated tests at present, too hard for us at the moment to figure out how
+    # to test a Blacklight extension in a reliable and sane way.
+    module SearchServiceBulkLoad
+      extend ActiveSupport::Concern
+
+      included do
+        class_attribute :bulk_load_records, default: true
+        class_attribute :bulk_load_scope
+      end
+
+      def search_results
+        (response, _documents) = super
+
+        if bulk_load_records
+          id_hash = response.documents.collect {|r| [r.id, r] }.to_h
+
+          scope = Kithe::Model.where(Kithe.indexable_settings.solr_id_value_attribute => id_hash.keys)
+          scope = scope.instance_exec(&bulk_load_scope) if bulk_load_scope
+
+          scope.find_each do |model|
+            id_hash[model.send(Kithe.indexable_settings.solr_id_value_attribute)].model = model
+          end
+
+          orphaned_solr_docs = id_hash.values.select { |doc| doc.model.nil? }
+          if orphaned_solr_docs.present?
+            Rails.logger.warn("Kithe::Blacklight::BulkLoading: Missing db records for solr doc id's: #{orphaned_solr_docs.collect(&:id).join(' ')}")
+          end
+        end
+
+        [response, _documents]
+      end
+    end
+  end
+end

data/lib/kithe/indexable_settings.rb

@@ -0,0 +1,34 @@
+module Kithe
+  class IndexableSettings
+    attr_accessor :solr_url, :writer_class_name, :writer_settings,
+                  :model_name_solr_field, :solr_id_value_attribute, :disable_callbacks
+    def initialize(solr_url:, writer_class_name:, writer_settings:,
+                   model_name_solr_field:, solr_id_value_attribute:, disable_callbacks: false)
+      @solr_url = solr_url
+      @writer_class_name = writer_class_name
+      @writer_settings = writer_settings
+      @model_name_solr_field = model_name_solr_field
+      @solr_id_value_attribute = solr_id_value_attribute
+    end
+
+    # Use configured solr_url, and merge together with configured
+    # writer_settings
+    def writer_settings
+      if solr_url
+        { "solr.url" => solr_url }.merge(@writer_settings)
+      else
+        @writer_settings
+      end
+    end
+
+    # Turn writer_class_name into an actual Class object.
+    def writer_class
+      writer_class_name.constantize
+    end
+
+    # Instantiate a new writer based on `writer_class_name` and `writer_settings`
+    def writer_instance!(additional_settings = {})
+      writer_class.new(writer_settings.merge(additional_settings))
+    end
+  end
+end