active-fedora 11.0.0 → 12.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +12 -2
  3. data/.rubocop_todo.yml +73 -18
  4. data/.travis.yml +8 -9
  5. data/CONTRIBUTING.md +66 -20
  6. data/History.txt +51 -0
  7. data/README.md +11 -14
  8. data/active-fedora.gemspec +14 -13
  9. data/lib/active_fedora/aggregation/base_extension.rb +3 -1
  10. data/lib/active_fedora/aggregation/list_source.rb +1 -1
  11. data/lib/active_fedora/associations/association.rb +1 -2
  12. data/lib/active_fedora/associations/basic_contains_association.rb +12 -0
  13. data/lib/active_fedora/associations/builder/association.rb +1 -2
  14. data/lib/active_fedora/associations/collection_association.rb +3 -3
  15. data/lib/active_fedora/associations/collection_proxy.rb +10 -10
  16. data/lib/active_fedora/associations/contained_finder.rb +2 -1
  17. data/lib/active_fedora/associations/has_and_belongs_to_many_association.rb +1 -1
  18. data/lib/active_fedora/associations/null_validator.rb +1 -2
  19. data/lib/active_fedora/associations/rdf.rb +1 -1
  20. data/lib/active_fedora/attached_files.rb +4 -0
  21. data/lib/active_fedora/attribute_methods.rb +1 -1
  22. data/lib/active_fedora/attributes/property_builder.rb +7 -5
  23. data/lib/active_fedora/caching_connection.rb +1 -1
  24. data/lib/active_fedora/cleaner.rb +1 -1
  25. data/lib/active_fedora/common.rb +10 -0
  26. data/lib/active_fedora/containers/container.rb +1 -2
  27. data/lib/active_fedora/core.rb +12 -5
  28. data/lib/active_fedora/errors.rb +4 -0
  29. data/lib/active_fedora/fedora.rb +19 -5
  30. data/lib/active_fedora/file/attributes.rb +5 -0
  31. data/lib/active_fedora/file.rb +3 -3
  32. data/lib/active_fedora/file_configurator.rb +5 -5
  33. data/lib/active_fedora/file_io.rb +120 -0
  34. data/lib/active_fedora/fixity_service.rb +36 -5
  35. data/lib/active_fedora/indexers/null_indexer.rb +1 -2
  36. data/lib/active_fedora/indexing/default_descriptors.rb +128 -0
  37. data/lib/active_fedora/indexing/descendant_fetcher.rb +105 -0
  38. data/lib/active_fedora/indexing/descriptor.rb +44 -0
  39. data/lib/active_fedora/indexing/field_mapper.rb +146 -0
  40. data/lib/active_fedora/indexing/inserter.rb +40 -0
  41. data/lib/active_fedora/indexing/map.rb +11 -3
  42. data/lib/active_fedora/indexing/suffix.rb +81 -0
  43. data/lib/active_fedora/indexing.rb +39 -15
  44. data/lib/active_fedora/indexing_service.rb +8 -3
  45. data/lib/active_fedora/inheritance.rb +1 -1
  46. data/lib/active_fedora/initializing_connection.rb +1 -1
  47. data/lib/active_fedora/ldp_resource.rb +1 -2
  48. data/lib/active_fedora/model_classifier.rb +1 -1
  49. data/lib/active_fedora/null_logger.rb +13 -0
  50. data/lib/active_fedora/orders/ordered_list.rb +1 -1
  51. data/lib/active_fedora/persistence.rb +16 -3
  52. data/lib/active_fedora/property.rb +1 -2
  53. data/lib/active_fedora/railtie.rb +6 -2
  54. data/lib/active_fedora/rake_support.rb +1 -1
  55. data/lib/active_fedora/rdf/field_map.rb +2 -2
  56. data/lib/active_fedora/rdf/field_map_entry.rb +15 -7
  57. data/lib/active_fedora/rdf/indexing_service.rb +15 -13
  58. data/lib/active_fedora/rdf/value_caster.rb +15 -0
  59. data/lib/active_fedora/rdf.rb +1 -0
  60. data/lib/active_fedora/relation/finder_methods.rb +28 -40
  61. data/lib/active_fedora/solr_hit.rb +1 -1
  62. data/lib/active_fedora/solr_service.rb +20 -2
  63. data/lib/active_fedora/type/value.rb +2 -4
  64. data/lib/active_fedora/version.rb +1 -1
  65. data/lib/active_fedora/with_metadata/metadata_node.rb +4 -1
  66. data/lib/active_fedora.rb +23 -9
  67. data/lib/generators/active_fedora/config/fedora/templates/.fcrepo_wrapper +1 -1
  68. data/lib/generators/active_fedora/config/solr/templates/solr/config/schema.xml +0 -6
  69. data/lib/generators/active_fedora/config/solr/templates/solr.yml +3 -3
  70. data/spec/integration/associations_spec.rb +8 -8
  71. data/spec/integration/attached_files_spec.rb +2 -3
  72. data/spec/integration/base_spec.rb +43 -39
  73. data/spec/integration/basic_contains_association_spec.rb +16 -3
  74. data/spec/integration/collection_association_spec.rb +1 -1
  75. data/spec/integration/direct_container_spec.rb +2 -2
  76. data/spec/integration/fedora_solr_sync_spec.rb +1 -1
  77. data/spec/integration/file_spec.rb +46 -13
  78. data/spec/integration/generators/fedora_generator_spec.rb +1 -1
  79. data/spec/integration/generators/solr_generator_spec.rb +1 -1
  80. data/spec/integration/has_and_belongs_to_many_associations_spec.rb +2 -2
  81. data/spec/integration/has_many_associations_spec.rb +4 -5
  82. data/spec/integration/indexing/descendant_fetcher_spec.rb +64 -0
  83. data/spec/integration/indexing_spec.rb +22 -2
  84. data/spec/integration/indirect_container_spec.rb +2 -2
  85. data/spec/integration/query_result_builder_spec.rb +2 -2
  86. data/spec/integration/scoping_spec.rb +20 -14
  87. data/spec/integration/solr_hit_spec.rb +1 -1
  88. data/spec/integration/versionable_spec.rb +2 -2
  89. data/spec/lib/active_fedora/null_logger_spec.rb +10 -0
  90. data/spec/unit/active_fedora/indexing/inserter_spec.rb +30 -0
  91. data/spec/unit/attached_files_spec.rb +36 -8
  92. data/spec/unit/attributes_spec.rb +8 -14
  93. data/spec/unit/base_spec.rb +7 -7
  94. data/spec/unit/builder/has_and_belongs_to_many_spec.rb +1 -1
  95. data/spec/unit/callback_spec.rb +2 -2
  96. data/spec/unit/change_set_spec.rb +1 -2
  97. data/spec/unit/collection_proxy_spec.rb +2 -1
  98. data/spec/unit/core/fedora_id_translator_spec.rb +2 -2
  99. data/spec/unit/core/fedora_uri_translator_spec.rb +2 -2
  100. data/spec/unit/core/logger_spec.rb +18 -0
  101. data/spec/unit/core_spec.rb +46 -12
  102. data/spec/unit/default_model_mapper_spec.rb +1 -1
  103. data/spec/unit/fedora_spec.rb +12 -0
  104. data/spec/unit/file_configurator_spec.rb +0 -9
  105. data/spec/unit/file_io_spec.rb +137 -0
  106. data/spec/unit/file_path_builder_spec.rb +3 -3
  107. data/spec/unit/file_spec.rb +70 -26
  108. data/spec/unit/files_hash_spec.rb +1 -2
  109. data/spec/unit/filter_spec.rb +2 -2
  110. data/spec/unit/finder_methods_spec.rb +1 -1
  111. data/spec/unit/fixity_service_spec.rb +65 -5
  112. data/spec/unit/has_and_belongs_to_many_association_spec.rb +1 -2
  113. data/spec/unit/has_many_association_spec.rb +2 -2
  114. data/spec/unit/indexing/map/index_object_spec.rb +25 -0
  115. data/spec/unit/indexing/map_spec.rb +27 -0
  116. data/spec/unit/indexing_spec.rb +3 -4
  117. data/spec/unit/inheritance_spec.rb +28 -1
  118. data/spec/unit/model_classifier_spec.rb +1 -1
  119. data/spec/unit/pathing_spec.rb +1 -2
  120. data/spec/unit/query_result_builder_spec.rb +1 -2
  121. data/spec/unit/query_spec.rb +4 -4
  122. data/spec/unit/rdf/indexing_service_spec.rb +18 -4
  123. data/spec/unit/reflection_spec.rb +2 -2
  124. data/spec/unit/solr_hit_spec.rb +1 -2
  125. data/spec/unit/solr_service_spec.rb +52 -4
  126. data/spec/unit/sparql_insert_spec.rb +1 -1
  127. data/spec/unit/validations_spec.rb +27 -4
  128. data/spec/unit/with_metadata/metadata_node_spec.rb +26 -1
  129. metadata +99 -29
@@ -0,0 +1,128 @@
1
+ module ActiveFedora
2
+ module Indexing
3
+ class DefaultDescriptors
4
+ # The suffix produced depends on the type parameter -- produces suffixes:
5
+ # _tesim - for strings or text fields
6
+ # _dtsim - for dates
7
+ # _isim - for integers
8
+ def self.stored_searchable
9
+ @stored_searchable ||= Descriptor.new(stored_searchable_field_definition, converter: searchable_converter, requires_type: true)
10
+ end
11
+
12
+ # The suffix produced depends on the type parameter -- produces suffixes:
13
+ # _teim - for strings or text fields
14
+ # _dtim - for dates
15
+ # _iim - for integers
16
+ def self.searchable
17
+ @searchable ||= Descriptor.new(searchable_field_definition, converter: searchable_converter, requires_type: true)
18
+ end
19
+
20
+ # Takes fields which are stored as strings, but we want indexed as dates. (e.g. "November 6th, 2012")
21
+ # produces suffixes:
22
+ # _dtsim - for dates
23
+ def self.dateable
24
+ @dateable ||= Descriptor.new(:date, :stored, :indexed, :multivalued, converter: dateable_converter)
25
+ end
26
+
27
+ # Produces _sim suffix
28
+ def self.facetable
29
+ @facetable ||= Descriptor.new(:string, :indexed, :multivalued)
30
+ end
31
+
32
+ # Produces _ssim suffix
33
+ # This is useful for when you only want to match whole words, such as user/group names from the the rightsMetadata datastream
34
+ def self.symbol
35
+ @symbol ||= Descriptor.new(:string, :stored, :indexed, :multivalued)
36
+ end
37
+
38
+ # The suffix produced depends on the type parameter -- produces suffixes:
39
+ # _tei - for text fields
40
+ # _si - for strings
41
+ # _dti - for dates
42
+ # _ii - for integers
43
+ def self.sortable
44
+ @sortable ||= Descriptor.new(sortable_field_definition, converter: searchable_converter, requires_type: true)
45
+ end
46
+
47
+ # Fields that are both stored and sortable
48
+ # Produces _ssi suffix if field_type is string
49
+ # Produces _dtsi suffix if field_type is date
50
+ def self.stored_sortable
51
+ @stored_sortable ||= Descriptor.new(->(field_type) { [field_type, :stored, :indexed] }, converter: searchable_converter)
52
+ end
53
+
54
+ # Produces _ssm suffix
55
+ def self.displayable
56
+ @displayable ||= Descriptor.new(:string, :stored, :multivalued)
57
+ end
58
+
59
+ # Produces _tim suffix (used to be _unstem)
60
+ def self.unstemmed_searchable
61
+ @unstemmed_searchable ||= Descriptor.new(:text, :indexed, :multivalued)
62
+ end
63
+
64
+ def self.simple
65
+ @simple ||= Descriptor.new(->(field_type) { [field_type, :indexed] })
66
+ end
67
+
68
+ class << self
69
+ def searchable_field_definition
70
+ lambda do |type|
71
+ type = :text_en if [:string, :text].include?(type) # for backwards compatibility with old solr schema
72
+ vals = [type, :indexed, :multivalued]
73
+ vals
74
+ end
75
+ end
76
+
77
+ def stored_searchable_field_definition
78
+ lambda do |type|
79
+ type = :text_en if [:string, :text].include?(type) # for backwards compatibility with old solr schema
80
+ if type == :boolean
81
+ [type, :indexed, :stored]
82
+ else
83
+ [type, :indexed, :stored, :multivalued]
84
+ end
85
+ end
86
+ end
87
+
88
+ def sortable_field_definition
89
+ lambda do |type|
90
+ vals = [type, :indexed]
91
+ vals
92
+ end
93
+ end
94
+
95
+ def searchable_converter
96
+ lambda do |type|
97
+ case type
98
+ when :date, :time
99
+ ->(val) { iso8601_date(val) }
100
+ end
101
+ end
102
+ end
103
+
104
+ def dateable_converter
105
+ lambda do |_type|
106
+ lambda do |val|
107
+ begin
108
+ iso8601_date(Date.parse(val))
109
+ rescue ArgumentError
110
+ nil
111
+ end
112
+ end
113
+ end
114
+ end
115
+
116
+ def iso8601_date(value)
117
+ if value.is_a?(Date) || value.is_a?(Time)
118
+ DateTime.parse(value.to_s).to_time.utc.strftime('%Y-%m-%dT%H:%M:%SZ')
119
+ elsif !value.empty?
120
+ DateTime.parse(value).to_time.utc.strftime('%Y-%m-%dT%H:%M:%SZ')
121
+ end
122
+ rescue ArgumentError
123
+ raise ArgumentError, "Unable to parse `#{value}' as a date-time object"
124
+ end
125
+ end
126
+ end
127
+ end
128
+ end
@@ -0,0 +1,105 @@
1
+ module ActiveFedora
2
+ module Indexing
3
+ # Finds all descendent URIs of a given repo URI (usually the base URI).
4
+ #
5
+ # This is a slow and non-performant thing to do, we need to fetch every single
6
+ # object from the repo.
7
+ #
8
+ # The DescendantFetcher is also capable of partitioning the URIs into "priority" URIs
9
+ # that will be first in the returned list. These prioritized URIs belong to objects
10
+ # with certain hasModel models. This feature is used in some samvera apps that need to
11
+ # index 'permissions' objects before other objects to have the solr indexing work right.
12
+ # And so by default, the prioritized class names are the ones form Hydra::AccessControls,
13
+ # but you can alter the prioritized model name list, or set it to the empty array.
14
+ #
15
+ # DescendantFetcher.new(ActiveFedora.fedora.base_uri).descendent_and_self_uris
16
+ # #=> array including self uri and descendent uris with "prioritized" (by default)
17
+ # Hydra::AccessControls permissions) objects FIRST.
18
+ #
19
+ # Change the default prioritized hasModel names:
20
+ #
21
+ # ActiveFedora::Indexing::DescendantFetcher.default_priority_models = []
22
+ class DescendantFetcher
23
+ HAS_MODEL_PREDICATE = ActiveFedora::RDF::Fcrepo::Model.hasModel
24
+
25
+ class_attribute :default_priority_models, instance_accessor: false
26
+ self.default_priority_models = %w(Hydra::AccessControl Hydra::AccessControl::Permissions).freeze
27
+
28
+ attr_reader :uri, :priority_models
29
+
30
+ def initialize(uri,
31
+ priority_models: self.class.default_priority_models, exclude_self: false)
32
+ @uri = uri
33
+ @priority_models = priority_models
34
+ @exclude_self = exclude_self
35
+ end
36
+
37
+ # @return [Array<String>] uris starting with priority models
38
+ def descendant_and_self_uris
39
+ partitioned = descendant_and_self_uris_partitioned
40
+ partitioned[:priority] + partitioned[:other]
41
+ end
42
+
43
+ # returns a hash where key :priority is an array of all prioritized
44
+ # type objects, key :other is an array of the rest.
45
+ # @return [Hash<String, Array<String>>] uris sorted into :priority and :other
46
+ def descendant_and_self_uris_partitioned
47
+ model_partitioned = descendant_and_self_uris_partitioned_by_model
48
+ { priority: model_partitioned.slice(*priority_models).values.flatten,
49
+ other: model_partitioned.slice(*(model_partitioned.keys - priority_models)).values.flatten }
50
+ end
51
+
52
+ # Returns a hash where keys are model names
53
+ # This is useful if you need to action on certain models and want finer grainularity than priority/other
54
+ # @return [Hash<String, Array<String>>] uris sorted by model names
55
+ def descendant_and_self_uris_partitioned_by_model
56
+ # GET could be slow if it's a big resource, we're using HEAD to avoid this problem,
57
+ # but this causes more requests to Fedora.
58
+ return partitioned_uris unless rdf_resource.head.rdf_source?
59
+
60
+ add_self_to_partitioned_uris unless @exclude_self
61
+
62
+ immediate_descendant_uris = rdf_graph.query(predicate: ::RDF::Vocab::LDP.contains).map { |descendant| descendant.object.to_s }
63
+ immediate_descendant_uris.each do |descendant_uri|
64
+ self.class.new(
65
+ descendant_uri,
66
+ priority_models: priority_models
67
+ ).descendant_and_self_uris_partitioned_by_model.tap do |descendant_partitioned|
68
+ descendant_partitioned.keys.each do |k|
69
+ partitioned_uris[k] ||= []
70
+ partitioned_uris[k].concat descendant_partitioned[k]
71
+ end
72
+ end
73
+ end
74
+ partitioned_uris
75
+ end
76
+
77
+ protected
78
+
79
+ def rdf_resource
80
+ @rdf_resource ||= Ldp::Resource::RdfSource.new(ActiveFedora.fedora.connection, uri)
81
+ end
82
+
83
+ def rdf_graph
84
+ @rdf_graph ||= rdf_resource.graph
85
+ end
86
+
87
+ def partitioned_uris
88
+ @partitioned_uris ||= {}
89
+ end
90
+
91
+ def rdf_graph_models
92
+ rdf_graph.query(predicate: HAS_MODEL_PREDICATE).collect(&:object).collect do |rdf_object|
93
+ rdf_object.to_s if rdf_object.literal?
94
+ end.compact
95
+ end
96
+
97
+ def add_self_to_partitioned_uris
98
+ rdf_graph_models.each do |model|
99
+ partitioned_uris[model] ||= []
100
+ partitioned_uris[model] << rdf_resource.subject
101
+ end
102
+ end
103
+ end
104
+ end
105
+ end
@@ -0,0 +1,44 @@
1
+ module ActiveFedora
2
+ module Indexing
3
+ class Descriptor
4
+ attr_reader :index_type
5
+ def initialize(*args)
6
+ if args.last.is_a? Hash
7
+ opts = args.pop
8
+ @converter = opts[:converter]
9
+ @type_required = opts[:requires_type]
10
+ end
11
+ @index_type = args
12
+ raise Solrizer::InvalidIndexDescriptor, "Invalid index type passed to Sorizer.solr_name. It should be an array like [:string, :indexed, :stored, :multivalued]. You provided: `#{@index_type}'" unless index_type.is_a? Array
13
+ end
14
+
15
+ def name_and_converter(field_name, args = nil)
16
+ args ||= {}
17
+ field_type = args[:type]
18
+ if type_required?
19
+ raise ArgumentError, "Must provide a :type argument when index_type is `#{self}' for #{field_name}" unless field_type
20
+ end
21
+ [field_name.to_s + suffix(field_type), converter(field_type)]
22
+ end
23
+
24
+ def type_required?
25
+ @type_required
26
+ end
27
+
28
+ def evaluate_suffix(field_type)
29
+ Suffix.new(index_type.first.is_a?(Proc) ? index_type.first.call(field_type) : index_type.dup)
30
+ end
31
+
32
+ protected
33
+
34
+ # Suffix can be overridden if you want a different method of grabbing the suffix
35
+ def suffix(field_type)
36
+ evaluate_suffix(field_type).to_s
37
+ end
38
+
39
+ def converter(field_type)
40
+ @converter.call(field_type) if @converter
41
+ end
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,146 @@
1
+ module ActiveFedora
2
+ module Indexing
3
+ class FieldMapper
4
+ class_attribute :id_field, :descriptors
5
+ # set defaults
6
+ self.id_field = 'id'
7
+ self.descriptors = [DefaultDescriptors]
8
+
9
+ # @api
10
+ # @params [Hash] doc the hash to insert the value into
11
+ # @params [String] name the name of the field (without the suffix)
12
+ # @params [String,Date] value the value to be inserted
13
+ # @params [Array,Hash] indexer_args the arguments that find the indexer
14
+ # @returns [Hash] doc the document that was provided with the new field (replacing any field with the same name)
15
+ def set_field(doc, name, value, *indexer_args)
16
+ # adding defaults indexer
17
+ indexer_args = [:stored_searchable] if indexer_args.empty?
18
+ doc.merge! solr_names_and_values(name, value, indexer_args)
19
+ doc
20
+ end
21
+
22
+ # @api
23
+ # Given a field name, index_type, etc., returns the corresponding Solr name.
24
+ # TODO field type is the input format, maybe we could just detect that?
25
+ # @param [String] field_name the ruby (term) name which will get a suffix appended to become a Solr field name
26
+ # @param opts - index_type is only needed if the FieldDescriptor requires it (e.g. :searcahble)
27
+ # @return [String] name of the solr field, based on the params
28
+ def solr_name(field_name, *opts)
29
+ index_type, args = if opts.first.is_a? Hash
30
+ [:stored_searchable, opts.first]
31
+ elsif opts.empty?
32
+ [:stored_searchable, { type: :text }]
33
+ else
34
+ [opts[0], opts[1] || { type: :string }]
35
+ end
36
+
37
+ indexer(index_type).name_and_converter(field_name, args).first
38
+ end
39
+
40
+ # Given a field name-value pair, a data type, and an array of index types, returns a hash of
41
+ # mapped names and values. The values in the hash are _arrays_, and may contain multiple values.
42
+ def solr_names_and_values(field_name, field_value, index_types)
43
+ return {} if field_value.nil?
44
+
45
+ # Determine the set of index types
46
+ index_types = Array(index_types)
47
+ index_types.uniq!
48
+ index_types.dup.each do |index_type|
49
+ if index_type.to_s =~ /^not_(.*)/
50
+ index_types.delete index_type # not_foo
51
+ index_types.delete Regexp.last_match(1).to_sym # foo
52
+ end
53
+ end
54
+
55
+ # Map names and values
56
+
57
+ results = {}
58
+
59
+ # Time seems to extend enumerable, so wrap it so we don't interate over each of its elements.
60
+ field_value = [field_value] if field_value.is_a? Time
61
+
62
+ index_types.each do |index_type|
63
+ Array(field_value).each do |single_value|
64
+ # Get mapping for field
65
+ descriptor = indexer(index_type)
66
+ data_type = extract_type(single_value)
67
+ name, converter = descriptor.name_and_converter(field_name, type: data_type)
68
+ next unless name
69
+
70
+ # Is there a custom converter?
71
+ # TODO instead of a custom converter, look for input data type and output data type. Create a few methods that can do that cast.
72
+
73
+ value = if converter
74
+ if converter.arity == 1
75
+ converter.call(single_value)
76
+ else
77
+ converter.call(single_value, field_name)
78
+ end
79
+ elsif data_type == :boolean
80
+ single_value
81
+ else
82
+ single_value.to_s
83
+ end
84
+
85
+ # Add mapped name & value, unless it's a duplicate
86
+ if descriptor.evaluate_suffix(data_type).multivalued?
87
+ values = (results[name] ||= [])
88
+ values << value unless value.nil? || values.include?(value)
89
+ else
90
+ Solrizer.logger.warn "Setting #{name} to `#{value}', but it already had `#{results[name]}'" if results[name] && Solrizer.logger
91
+ results[name] = value
92
+ end
93
+ end
94
+ end
95
+
96
+ results
97
+ end
98
+
99
+ private
100
+
101
+ # @param [Symbol, String, Descriptor] index_type is a Descriptor, a symbol that references a method that returns a Descriptor, or a string which will be used as the suffix.
102
+ # @return [Descriptor]
103
+ def indexer(index_type)
104
+ index_type = case index_type
105
+ when Symbol
106
+ index_type_macro(index_type)
107
+ when String
108
+ StringDescriptor.new(index_type)
109
+ when Descriptor
110
+ index_type
111
+ else
112
+ raise Solrizer::InvalidIndexDescriptor, "#{index_type.class} is not a valid indexer_type. Use a String, Symbol or Descriptor."
113
+ end
114
+
115
+ raise InvalidIndexDescriptor, "index type should be an Descriptor, you passed: #{index_type.class}" unless index_type.is_a? Descriptor
116
+ index_type
117
+ end
118
+
119
+ # @param index_type [Symbol]
120
+ # search through the descriptors (class attribute) until a module is found that responds to index_type, then call it.
121
+ def index_type_macro(index_type)
122
+ klass = self.class.descriptors.find { |descriptor_klass| descriptor_klass.respond_to? index_type }
123
+ if klass
124
+ klass.send(index_type)
125
+ else
126
+ raise UnknownIndexMacro, "Unable to find `#{index_type}' in #{self.class.descriptors}"
127
+ end
128
+ end
129
+
130
+ def extract_type(value)
131
+ case value
132
+ when NilClass
133
+ nil
134
+ when Integer # In ruby < 2.4, Fixnum extends Integer
135
+ :integer
136
+ when DateTime
137
+ :time
138
+ when TrueClass, FalseClass
139
+ :boolean
140
+ else
141
+ value.class.to_s.underscore.to_sym
142
+ end
143
+ end
144
+ end
145
+ end
146
+ end
@@ -0,0 +1,40 @@
1
+ module ActiveFedora
2
+ module Indexing
3
+ # Utilities for adding fields to index documents
4
+ class Inserter
5
+ # @param [String] field_name_base the field name
6
+ # @param [String] value the value to insert into the index
7
+ # @param [Array<Symbol>] index_as the index type suffixes
8
+ # @param [Hash] solr_doc the index doc to add to
9
+ # @example:
10
+ # solr_doc = {}
11
+ # create_and_insert_terms('title', 'War and Peace', [:displayable, :searchable], solr_doc)
12
+ # solr_doc
13
+ # # => {"title_ssm"=>["War and Peace"], "title_teim"=>["War and Peace"]}
14
+ def self.create_and_insert_terms(field_name_base, value, index_as, solr_doc)
15
+ index_as.each do |indexer|
16
+ insert_field(solr_doc, field_name_base, value, indexer)
17
+ end
18
+ end
19
+
20
+ # @params [Hash] doc the hash to insert the value into
21
+ # @params [String] name the name of the field (without the suffix)
22
+ # @params [String,Date,Array] value the value (or array of values) to be inserted
23
+ # @params [Array,Hash] indexer_args the arguments that find the indexer
24
+ # @returns [Hash] doc the document that was provided with the new field inserted
25
+ def self.insert_field(doc, name, value, *indexer_args)
26
+ # adding defaults indexer
27
+ indexer_args = [:stored_searchable] if indexer_args.empty?
28
+ ActiveFedora.index_field_mapper.solr_names_and_values(name, value, indexer_args).each do |k, v|
29
+ doc[k] ||= []
30
+ if v.is_a? Array
31
+ doc[k] += v
32
+ else
33
+ doc[k] = v
34
+ end
35
+ end
36
+ doc
37
+ end
38
+ end
39
+ end
40
+ end
@@ -11,7 +11,15 @@ module ActiveFedora::Indexing
11
11
  end
12
12
 
13
13
  def dup
14
- self.class.new(@hash.deep_dup)
14
+ self.class.new(to_hash)
15
+ end
16
+
17
+ def merge(new_hash)
18
+ self.class.new(to_hash.merge(new_hash))
19
+ end
20
+
21
+ def to_hash
22
+ @hash.deep_dup
15
23
  end
16
24
 
17
25
  # this enables a cleaner API for solr integration
@@ -19,8 +27,8 @@ module ActiveFedora::Indexing
19
27
  attr_accessor :data_type, :behaviors, :term
20
28
  attr_reader :key
21
29
 
22
- def initialize(name, &_block)
23
- @behaviors = []
30
+ def initialize(name, behaviors: [], &_block)
31
+ @behaviors = behaviors
24
32
  @data_type = :string
25
33
  @key = name
26
34
  yield self if block_given?
@@ -0,0 +1,81 @@
1
+ require 'ostruct'
2
+
3
+ module ActiveFedora
4
+ module Indexing
5
+ class Suffix
6
+ def initialize(*fields)
7
+ @fields = fields.flatten
8
+ end
9
+
10
+ def multivalued?
11
+ has_field? :multivalued
12
+ end
13
+
14
+ def stored?
15
+ has_field? :stored
16
+ end
17
+
18
+ def indexed?
19
+ has_field? :indexed
20
+ end
21
+
22
+ def has_field?(f)
23
+ (f.to_sym == :type) || @fields.include?(f.to_sym)
24
+ end
25
+
26
+ def data_type
27
+ @fields.first
28
+ end
29
+
30
+ def to_s
31
+ raise Solrizer::InvalidIndexDescriptor, "Missing datatype for #{@fields}" unless data_type
32
+
33
+ field_suffix = [config.suffix_delimiter]
34
+
35
+ config.fields.select { |f| has_field? f }.each do |f|
36
+ key = :"#{f}_suffix"
37
+ field_suffix << if config.send(key).is_a? Proc
38
+ config.send(key).call(@fields)
39
+ else
40
+ config.send(key)
41
+ end
42
+ end
43
+
44
+ field_suffix.join
45
+ end
46
+
47
+ def self.config
48
+ @config ||= OpenStruct.new fields: [:type, :stored, :indexed, :multivalued],
49
+ suffix_delimiter: '_',
50
+ type_suffix: (lambda do |fields|
51
+ type = fields.first
52
+ case type
53
+ when :string, :symbol # TODO: `:symbol' usage ought to be deprecated
54
+ 's'
55
+ when :text
56
+ 't'
57
+ when :text_en
58
+ 'te'
59
+ when :date, :time
60
+ 'dt'
61
+ when :integer
62
+ 'i'
63
+ when :boolean
64
+ 'b'
65
+ when :long
66
+ 'lt'
67
+ else
68
+ raise Solrizer::InvalidIndexDescriptor, "Invalid datatype `#{type.inspect}'. Must be one of: :date, :time, :text, :text_en, :string, :symbol, :integer, :boolean"
69
+ end
70
+ end),
71
+ stored_suffix: 's',
72
+ indexed_suffix: 'i',
73
+ multivalued_suffix: 'm'
74
+ end
75
+
76
+ def config
77
+ @config ||= self.class.config.dup
78
+ end
79
+ end
80
+ end
81
+ end
@@ -12,7 +12,13 @@ module ActiveFedora
12
12
  extend ActiveSupport::Autoload
13
13
 
14
14
  eager_autoload do
15
+ autoload :DefaultDescriptors
16
+ autoload :Descriptor
17
+ autoload :DescendantFetcher
18
+ autoload :FieldMapper
19
+ autoload :Inserter
15
20
  autoload :Map
21
+ autoload :Suffix
16
22
  end
17
23
 
18
24
  included do
@@ -82,26 +88,44 @@ module ActiveFedora
82
88
  end
83
89
  end
84
90
 
85
- def reindex_everything
86
- descendants = descendant_uris(ActiveFedora::Base.id_to_uri(''))
87
- descendants.shift # Discard the root uri
91
+ # @param [Integer] batch_size - The number of Fedora objects to process for each SolrService.add call. Default 50.
92
+ # @param [Boolean] softCommit - Do we perform a softCommit when we add the to_solr objects to SolrService. Default true.
93
+ # @param [Boolean] progress_bar - If true output progress bar information. Default false.
94
+ # @param [Boolean] final_commit - If true perform a hard commit to the Solr service at the completion of the batch of updates. Default false.
95
+ def reindex_everything(batch_size: 50, softCommit: true, progress_bar: false, final_commit: false)
96
+ # skip root url
97
+ descendants = descendant_uris(ActiveFedora.fedora.base_uri, exclude_uri: true)
98
+
99
+ batch = []
100
+
101
+ progress_bar_controller = ProgressBar.create(total: descendants.count, format: "%t: |%B| %p%% %e") if progress_bar
102
+
88
103
  descendants.each do |uri|
89
104
  logger.debug "Re-index everything ... #{uri}"
90
- ActiveFedora::Base.find(ActiveFedora::Base.uri_to_id(uri)).update_index
105
+
106
+ batch << ActiveFedora::Base.find(ActiveFedora::Base.uri_to_id(uri)).to_solr
107
+
108
+ if (batch.count % batch_size).zero?
109
+ SolrService.add(batch, softCommit: softCommit)
110
+ batch.clear
111
+ end
112
+
113
+ progress_bar_controller.increment if progress_bar_controller
114
+ end
115
+
116
+ if batch.present?
117
+ SolrService.add(batch, softCommit: softCommit)
118
+ batch.clear
91
119
  end
92
- end
93
120
 
94
- def descendant_uris(uri)
95
- resource = Ldp::Resource::RdfSource.new(ActiveFedora.fedora.connection, uri)
96
- # GET could be slow if it's a big resource, we're using HEAD to avoid this problem,
97
- # but this causes more requests to Fedora.
98
- return [] unless resource.head.rdf_source?
99
- immediate_descendant_uris = resource.graph.query(predicate: ::RDF::Vocab::LDP.contains).map { |descendant| descendant.object.to_s }
100
- all_descendants_uris = [uri]
101
- immediate_descendant_uris.each do |descendant_uri|
102
- all_descendants_uris += descendant_uris(descendant_uri)
121
+ if final_commit
122
+ logger.debug "Solr hard commit..."
123
+ SolrService.commit
103
124
  end
104
- all_descendants_uris
125
+ end
126
+
127
+ def descendant_uris(uri, exclude_uri: false)
128
+ DescendantFetcher.new(uri, exclude_self: exclude_uri).descendant_and_self_uris
105
129
  end
106
130
  end
107
131
  end
@@ -32,8 +32,8 @@ module ActiveFedora
32
32
  # @return [Hash] the solr document
33
33
  def generate_solr_document
34
34
  solr_doc = {}
35
- Solrizer.set_field(solr_doc, 'system_create', c_time, :stored_sortable)
36
- Solrizer.set_field(solr_doc, 'system_modified', m_time, :stored_sortable)
35
+ ActiveFedora.index_field_mapper.set_field(solr_doc, 'system_create', c_time, :stored_sortable)
36
+ ActiveFedora.index_field_mapper.set_field(solr_doc, 'system_modified', m_time, :stored_sortable)
37
37
  solr_doc[QueryResultBuilder::HAS_MODEL_SOLR_FIELD] = object.has_model
38
38
  solr_doc[ActiveFedora.id_field.to_sym] = object.id
39
39
  object.declared_attached_files.each do |name, file|
@@ -61,7 +61,12 @@ module ActiveFedora
61
61
  # Serialize the resource's RDF relationships to solr
62
62
  # @param [Hash] solr_doc @deafult an empty Hash
63
63
  def solrize_rdf_assertions(solr_doc = {})
64
- solr_doc.merge rdf_service.new(object).generate_solr_document
64
+ solr_doc.merge rdf_indexer.generate_solr_document
65
+ end
66
+
67
+ # @return IndexingService
68
+ def rdf_indexer
69
+ rdf_service.new(object, object.class.index_config)
65
70
  end
66
71
  end
67
72
  end
@@ -12,7 +12,7 @@ module ActiveFedora
12
12
  # If B < A and C < B and if A is an abstract_class then both B.base_class
13
13
  # and C.base_class would return B as the answer since A is an abstract_class.
14
14
  def base_class
15
- return File if self == File || superclass == File
15
+ return File if self <= File
16
16
 
17
17
  unless self <= Base
18
18
  raise ActiveFedoraError, "#{name} doesn't belong in a hierarchy descending from ActiveFedora"