active-fedora 11.1.3 → 11.2.1

data/lib/active_fedora/indexing.rb

@@ -13,6 +13,7 @@ module ActiveFedora
 
     eager_autoload do
       autoload :Map
+      autoload :DescendantFetcher
     end
 
     included do
@@ -82,26 +83,44 @@ module ActiveFedora
                             end
         end
 
-        def reindex_everything
-          descendants = descendant_uris(ActiveFedora.fedora.base_uri)
-          descendants.shift # Discard the root uri
+        # @param [Integer] batch_size - The number of Fedora objects to process for each SolrService.add call. Default 50.
+        # @param [Boolean] softCommit - Do we perform a softCommit when we add the to_solr objects to SolrService. Default true.
+        # @param [Boolean] progress_bar - If true output progress bar information. Default false.
+        # @param [Boolean] final_commit - If true perform a hard commit to the Solr service at the completion of the batch of updates. Default false.
+        def reindex_everything(batch_size: 50, softCommit: true, progress_bar: false, final_commit: false)
+          # skip root url
+          descendants = descendant_uris(ActiveFedora.fedora.base_uri, exclude_uri: true)
+
+          batch = []
+
+          progress_bar_controller = ProgressBar.create(total: descendants.count, format: "%t: |%B| %p%% %e") if progress_bar
+
           descendants.each do |uri|
             logger.debug "Re-index everything ... #{uri}"
-            ActiveFedora::Base.find(ActiveFedora::Base.uri_to_id(uri)).update_index
+
+            batch << ActiveFedora::Base.find(ActiveFedora::Base.uri_to_id(uri)).to_solr
+
+            if (batch.count % batch_size).zero?
+              SolrService.add(batch, softCommit: softCommit)
+              batch.clear
+            end
+
+            progress_bar_controller.increment if progress_bar_controller
+          end
+
+          if batch.present?
+            SolrService.add(batch, softCommit: softCommit)
+            batch.clear
           end
-        end
 
-        def descendant_uris(uri)
-          resource = Ldp::Resource::RdfSource.new(ActiveFedora.fedora.connection, uri)
-          # GET could be slow if it's a big resource, we're using HEAD to avoid this problem,
-          # but this causes more requests to Fedora.
-          return [] unless resource.head.rdf_source?
-          immediate_descendant_uris = resource.graph.query(predicate: ::RDF::Vocab::LDP.contains).map { |descendant| descendant.object.to_s }
-          all_descendants_uris = [uri]
-          immediate_descendant_uris.each do |descendant_uri|
-            all_descendants_uris += descendant_uris(descendant_uri)
+          if final_commit
+            logger.debug "Solr hard commit..."
+            SolrService.commit
           end
-          all_descendants_uris
+        end
+
+        def descendant_uris(uri, exclude_uri: false)
+          DescendantFetcher.new(uri, exclude_self: exclude_uri).descendant_and_self_uris
         end
       end
   end

data/lib/active_fedora/indexing/descendant_fetcher.rb

@@ -0,0 +1,101 @@
+module ActiveFedora
+  module Indexing
+    # Finds all descendent URIs of a given repo URI (usually the base URI).
+    #
+    # This is a slow and non-performant thing to do, we need to fetch every single
+    # object from the repo.
+    #
+    # The DescendantFetcher is also capable of partitioning the URIs into "priority" URIs
+    # that will be first in the returned list. These prioritized URIs belong to objects
+    # with certain hasModel models. This feature is used in some hydra apps that need to
+    # index 'permissions' objects before other objects to have the solr indexing work right.
+    # And so by default, the prioritized class names are the ones form Hydra::AccessControls,
+    # but you can alter the prioritized model name list, or set it to the empty array.
+    #
+    #     DescendantFetcher.new(ActiveFedora.fedora.base_uri).descendent_and_self_uris
+    #     #=> array including self uri and descendent uris with "prioritized" (by default)
+    #         Hydra::AccessControls permissions) objects FIRST.
+    #
+    # Change the default prioritized hasModel names:
+    #
+    #     ActiveFedora::Indexing::DescendantFetcher.default_priority_models = []
+    class DescendantFetcher
+      HAS_MODEL_PREDICATE = ActiveFedora::RDF::Fcrepo::Model.hasModel
+
+      class_attribute :default_priority_models, instance_accessor: false
+      self.default_priority_models = %w(Hydra::AccessControl Hydra::AccessControl::Permissions).freeze
+
+      attr_reader :uri, :priority_models
+
+      def initialize(uri,
+                     priority_models: self.class.default_priority_models, exclude_self: false)
+        @uri = uri
+        @priority_models = priority_models
+        @exclude_self = exclude_self
+      end
+
+      def descendant_and_self_uris
+        partitioned = descendant_and_self_uris_partitioned
+        partitioned[:priority] + partitioned[:other]
+      end
+
+      # returns a hash where key :priority is an array of all prioritized
+      # type objects, key :other is an array of the rest.
+      def descendant_and_self_uris_partitioned
+        resource = Ldp::Resource::RdfSource.new(ActiveFedora.fedora.connection, uri)
+        # GET could be slow if it's a big resource, we're using HEAD to avoid this problem,
+        # but this causes more requests to Fedora.
+        return partitioned_uris unless resource.head.rdf_source?
+
+        add_self_to_partitioned_uris unless @exclude_self
+
+        immediate_descendant_uris = rdf_graph.query(predicate: ::RDF::Vocab::LDP.contains).map { |descendant| descendant.object.to_s }
+        immediate_descendant_uris.each do |descendant_uri|
+          self.class.new(
+            descendant_uri,
+            priority_models: priority_models
+          ).descendant_and_self_uris_partitioned.tap do |descendant_partitioned|
+            partitioned_uris[:priority].concat descendant_partitioned[:priority]
+            partitioned_uris[:other].concat descendant_partitioned[:other]
+          end
+        end
+        partitioned_uris
+      end
+
+      protected
+
+        def rdf_resource
+          @rdf_resource ||= Ldp::Resource::RdfSource.new(ActiveFedora.fedora.connection, uri)
+        end
+
+        def rdf_graph
+          @rdf_graph ||= rdf_resource.graph
+        end
+
+        def partitioned_uris
+          @partitioned_uris ||= {
+            priority: [],
+            other: []
+          }
+        end
+
+        def rdf_graph_models
+          rdf_graph.query(predicate: HAS_MODEL_PREDICATE).collect(&:object).collect do |rdf_object|
+            rdf_object.to_s if rdf_object.literal?
+          end.compact
+        end
+
+        def prioritized_object?
+          priority_models.present? && (rdf_graph_models & priority_models).count > 0
+        end
+
+        def add_self_to_partitioned_uris
+          if prioritized_object?
+            partitioned_uris[:priority] << rdf_resource.subject
+          else
+            partitioned_uris[:other] << rdf_resource.subject
+          end
+        end
+    end
+  end
+end

data/lib/active_fedora/indexing/map.rb

@@ -11,7 +11,15 @@ module ActiveFedora::Indexing
     end
 
     def dup
-      self.class.new(@hash.deep_dup)
+      self.class.new(to_hash)
+    end
+
+    def merge(new_hash)
+      self.class.new(to_hash.merge(new_hash))
+    end
+
+    def to_hash
+      @hash.deep_dup
     end
 
     # this enables a cleaner API for solr integration
@@ -19,8 +27,8 @@ module ActiveFedora::Indexing
       attr_accessor :data_type, :behaviors, :term
       attr_reader :key
 
-      def initialize(name, &_block)
-        @behaviors = []
+      def initialize(name, behaviors: [], &_block)
+        @behaviors = behaviors
         @data_type = :string
         @key = name
         yield self if block_given?

data/lib/active_fedora/indexing_service.rb

@@ -61,7 +61,12 @@ module ActiveFedora
       # Serialize the resource's RDF relationships to solr
       # @param [Hash] solr_doc @deafult an empty Hash
       def solrize_rdf_assertions(solr_doc = {})
-        solr_doc.merge rdf_service.new(object).generate_solr_document
+        solr_doc.merge rdf_indexer.generate_solr_document
+      end
+
+      # @return IndexingService
+      def rdf_indexer
+        rdf_service.new(object, object.class.index_config)
       end
   end
 end

data/lib/active_fedora/inheritance.rb

@@ -12,7 +12,7 @@ module ActiveFedora
       # If B < A and C < B and if A is an abstract_class then both B.base_class
       # and C.base_class would return B as the answer since A is an abstract_class.
       def base_class
-        return File if self == File || superclass == File
+        return File if self <= File
 
         unless self <= Base
           raise ActiveFedoraError, "#{name} doesn't belong in a hierarchy descending from ActiveFedora"

data/lib/active_fedora/null_logger.rb

@@ -1,11 +1,9 @@
 module ActiveFedora
   class NullLogger < Logger
-    def initialize(*)
-    end
+    def initialize(*); end
 
     # allows all the usual logger method calls (warn, info, error, etc.)
-    def add(*)
-    end
+    def add(*); end
 
     # In the NullLogger there are no levels, so none of these should be true.
     [:debug?, :info?, :warn?, :error?, :fatal?].each do |method_name|

data/lib/active_fedora/persistence.rb

@@ -205,8 +205,7 @@ module ActiveFedora
       end
 
       # Override to tie in an ID minting service
-      def assign_id
-      end
+      def assign_id; end
 
       # This is only used when creating a new record. If the object doesn't have an id
       # and assign_id can mint an id for the object, then assign it to the resource.
@@ -228,6 +227,7 @@ module ActiveFedora
         root_resource_path
       end
 
+      # Check to see if the :base_path (from fedora.yml) exists in Fedora. If it doesn't exist, then create it.
       def init_root_path
         path = root_resource_path.gsub(/^\//, "")
         ActiveFedora.fedora.connection.head(path)

data/lib/active_fedora/property.rb

@@ -7,7 +7,6 @@ module ActiveFedora
       @instance_variable_name = "@#{@name}"
     end
 
-    def field
-    end
+    def field; end
   end
 end

data/lib/active_fedora/rake_support.rb

@@ -12,7 +12,7 @@ def with_server(environment)
   # setting port: nil assigns a random port.
   solr_defaults = { port: nil, verbose: true, managed: true }
   fcrepo_defaults = { port: nil, verbose: true, managed: true,
-                      enable_jms: false, fcrepo_home_dir: "fcrepo4-#{environment}-data" }
+                      enable_jms: false, fcrepo_home_dir: "tmp/fcrepo4-#{environment}-data" }
 
   SolrWrapper.wrap(load_config(:solr, environment, solr_defaults)) do |solr|
     ENV["SOLR_#{environment.upcase}_PORT"] = solr.port.to_s

data/lib/active_fedora/rdf/field_map.rb

@@ -15,8 +15,8 @@ module ActiveFedora::RDF
 
     # Inserts each solr field map configuration into the FieldMap class
     # @param [Symbol] name the name of the property on the object that we're indexing
-    # @param [Object] index_field_config an instance of ActiveFedora::Indexing::Map::IndexObject
-    # @param [Object] object the instance of ActiveFedora::Base which is being indexed into Solr
+    # @param [ActiveFedora::Indexing::Map::IndexObject] index_field_config describes how the object should be indexed
+    # @param [ActiveFedora::Base] object the object to be indexed into Solr
     def insert(name, index_field_config, object)
       self[index_field_config.key.to_s] ||= FieldMapEntry.new
       PolymorphicBuilder.new(self[index_field_config.key.to_s], index_field_config, object, name).build

data/lib/active_fedora/rdf/field_map_entry.rb

@@ -3,9 +3,10 @@ module ActiveFedora::RDF
   # It might be possible for two properties to share a single field map entry if they use the same solr key.
   # @attribute [Symbol] type the data type hint for Solrizer
   # @attribute [Array] behaviors the indexing hints such as :stored_searchable or :symbol
-  # @attribute [Array] values the actual values that get sent to solr
+  # @!attribute [w] values the raw values
   class FieldMapEntry
-    attr_accessor :type, :behaviors, :values
+    attr_accessor :type, :behaviors
+    attr_writer :values
 
     def initialize
       @behaviors = []
@@ -15,14 +16,15 @@ module ActiveFedora::RDF
     # Merges any existing values for solr fields with new, incoming values and ensures that resulting values are unique.
     # @param [Symbol] type the data type for the field such as :string, :date, :integer
     # @param [Array] behaviors Solrizer's behaviors for indexing such as :stored_searhable, :symbol
-    # @param [Array] values existing values for the solr field
-    def merge!(type, behaviors, values)
+    # @param [Array] new_values values to append into the existing solr field
+    def merge!(type, behaviors, new_values)
       self.type ||= type
       self.behaviors += behaviors
       self.behaviors.uniq!
-      self.values += values
+      self.values += new_values
     end
 
+    # @return [Array] the actual values that get sent to solr
     def values
       @values.map do |value|
         ValueCaster.new(value).value

data/lib/active_fedora/rdf/indexing_service.rb

@@ -6,11 +6,17 @@ module ActiveFedora::RDF
   # @see ActiveFedora::IndexingService
   class IndexingService
     include Solrizer::Common
-    attr_reader :object
+    attr_reader :object, :index_config
 
-    # @param [#resource, #rdf_subject] obj the object to build an solr document for. Its class must respond to 'properties' and 'index_config'
-    def initialize(obj)
+    # @param [#resource, #rdf_subject] obj the object to build an solr document for. Its class must respond to 'properties'
+    # @param [ActiveFedora::Indexing::Map] index_config the configuration to use to map object values to index document values
+    def initialize(obj, index_config = nil)
+      unless index_config
+        Deprecation.warn(self, "initializing ActiveFedora::RDF::IndexingService without an index_config is deprecated and will be removed in ActiveFedora 13.0")
+        index_config = obj.class.index_config
+      end
       @object = obj
+      @index_config = index_config
     end
 
     # Creates a solr document hash for the rdf assertions of the {#object}
@@ -69,10 +75,6 @@ module ActiveFedora::RDF
         object.resource
       end
 
-      def index_config
-        object.class.index_config
-      end
-
       # returns the field map instance
       def fields
         field_map_class.new do |field_map|

data/lib/active_fedora/relation/finder_methods.rb

@@ -189,7 +189,8 @@ module ActiveFedora
     protected
 
       def load_from_fedora(id, cast)
-        raise ActiveFedora::ObjectNotFoundError if id.empty?
+        raise ActiveFedora::ObjectNotFoundError, "No ID provided for #{klass.name}." if id.empty?
+
         resource = ActiveFedora.fedora.ldp_resource_service.build(klass, id)
         raise_record_not_found_exception!(id) if resource.new?
         class_to_load(resource, cast).allocate.init_with_resource(resource) # Triggers the find callback

data/lib/active_fedora/solr_service.rb

@@ -2,6 +2,7 @@ require 'rsolr'
 
 module ActiveFedora
   class SolrService
+    attr_reader :options
     attr_writer :conn
 
     MAX_ROWS = 10_000
@@ -64,7 +65,7 @@ module ActiveFedora
         SolrService.get(query, args)['response']['numFound'].to_i
       end
 
-      # @param [Hash] doc the document to index
+      # @param [Hash] doc the document to index, or an array of docs
       # @param [Hash] params
       #   :commit => commits immediately
       #   :softCommit => commit to memory, but don't flush to disk

data/lib/active_fedora/type/value.rb

@@ -11,8 +11,7 @@ module ActiveFedora
         @limit = limit
       end
 
-      def type # :nodoc:
-      end
+      def type; end
 
       # Converts a value from database input to the appropriate ruby type. The
       # return value of this method will be returned from
@@ -102,8 +101,7 @@ module ActiveFedora
         [self.class, precision, scale, limit].hash
       end
 
-      def assert_valid_value(*)
-      end
+      def assert_valid_value(*); end
 
       private
 

data/lib/active_fedora/version.rb

@@ -1,3 +1,3 @@
 module ActiveFedora
-  VERSION = "11.1.3".freeze
+  VERSION = "11.2.1".freeze
 end

data/lib/active_fedora/with_metadata/metadata_node.rb

@@ -36,6 +36,7 @@ module ActiveFedora
       end
 
       def ldp_source
+        @ldp_source ||= LdpResource.new(ldp_connection, nil) if file.new_record?
         @ldp_source ||= LdpResource.new(ldp_connection, metadata_uri)
       end
 
@@ -47,6 +48,7 @@ module ActiveFedora
         raise "Save the file first" if file.new_record?
         SparqlInsert.new(changes_for_update, file.uri).execute(metadata_uri)
         @ldp_source = nil
+        @metadata_uri = nil
         true
       end
 

data/spec/integration/associations_spec.rb

@@ -16,8 +16,8 @@ describe ActiveFedora::Base do
       Object.send(:remove_const, :Person)
     end
 
-    let(:person) { Person.create }
     subject(:book) { Book.new(author: person) }
+    let(:person) { Person.create }
 
     it "goes" do
       book.save
@@ -511,8 +511,7 @@ describe ActiveFedora::Base do
             after_count(m.reload.library_books.count)
           end
 
-          def say_hi(_var)
-          end
+          def say_hi(_var); end
         end
         class Page < ActiveFedora::Base
           has_many :library_books, predicate: ActiveFedora::RDF::Fcrepo::RelsExt.isPartOf
@@ -524,14 +523,15 @@ describe ActiveFedora::Base do
       end
 
       describe "removing association" do
-        let(:p1) { Page.create }
-        let(:p2) { Page.create }
         subject(:book) do
           book = LibraryBook.create
           book.pages << p1 << p2
           book.save!
           book
         end
+        let(:p1) { Page.create }
+        let(:p2) { Page.create }
+
         it "saves between the before and after hooks" do
           expect(book).to receive(:before_count).with(1)
           expect(book).to receive(:after_count).with(0)
@@ -582,7 +582,7 @@ describe ActiveFedora::Base do
         Object.send(:remove_const, :MediaObject)
       end
 
-      it "it should find the reflection that bears the predicate" do
+      it "finds the reflection that bears the predicate" do
         expect(MediaObject.new.association(:baubles).send(:find_reflection)).to eq Bauble._reflect_on_association(:media_object)
       end
     end
@@ -602,7 +602,7 @@ describe ActiveFedora::Base do
         Object.send(:remove_const, :MediaObject)
       end
 
-      it "it should find the reflection that bears the predicate" do
+      it "finds the reflection that bears the predicate" do
         expect(MediaObject.new.association(:parts).send(:find_reflection)).to eq MasterFile._reflect_on_association(:media_object)
       end
     end
@@ -622,7 +622,7 @@ describe ActiveFedora::Base do
         Object.send(:remove_const, :MediaObject)
       end
 
-      it "it should find the reflection that bears the predicate" do
+      it "finds the reflection that bears the predicate" do
         expect(MediaObject.new.association(:baubles).send(:find_reflection)).to eq MediaObject._reflect_on_association(:baubles)
       end
     end