active-fedora 11.0.0 → 12.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +12 -2
- data/.rubocop_todo.yml +73 -18
- data/.travis.yml +8 -9
- data/CONTRIBUTING.md +66 -20
- data/History.txt +51 -0
- data/README.md +11 -14
- data/active-fedora.gemspec +14 -13
- data/lib/active_fedora/aggregation/base_extension.rb +3 -1
- data/lib/active_fedora/aggregation/list_source.rb +1 -1
- data/lib/active_fedora/associations/association.rb +1 -2
- data/lib/active_fedora/associations/basic_contains_association.rb +12 -0
- data/lib/active_fedora/associations/builder/association.rb +1 -2
- data/lib/active_fedora/associations/collection_association.rb +3 -3
- data/lib/active_fedora/associations/collection_proxy.rb +10 -10
- data/lib/active_fedora/associations/contained_finder.rb +2 -1
- data/lib/active_fedora/associations/has_and_belongs_to_many_association.rb +1 -1
- data/lib/active_fedora/associations/null_validator.rb +1 -2
- data/lib/active_fedora/associations/rdf.rb +1 -1
- data/lib/active_fedora/attached_files.rb +4 -0
- data/lib/active_fedora/attribute_methods.rb +1 -1
- data/lib/active_fedora/attributes/property_builder.rb +7 -5
- data/lib/active_fedora/caching_connection.rb +1 -1
- data/lib/active_fedora/cleaner.rb +1 -1
- data/lib/active_fedora/common.rb +10 -0
- data/lib/active_fedora/containers/container.rb +1 -2
- data/lib/active_fedora/core.rb +12 -5
- data/lib/active_fedora/errors.rb +4 -0
- data/lib/active_fedora/fedora.rb +19 -5
- data/lib/active_fedora/file/attributes.rb +5 -0
- data/lib/active_fedora/file.rb +3 -3
- data/lib/active_fedora/file_configurator.rb +5 -5
- data/lib/active_fedora/file_io.rb +120 -0
- data/lib/active_fedora/fixity_service.rb +36 -5
- data/lib/active_fedora/indexers/null_indexer.rb +1 -2
- data/lib/active_fedora/indexing/default_descriptors.rb +128 -0
- data/lib/active_fedora/indexing/descendant_fetcher.rb +105 -0
- data/lib/active_fedora/indexing/descriptor.rb +44 -0
- data/lib/active_fedora/indexing/field_mapper.rb +146 -0
- data/lib/active_fedora/indexing/inserter.rb +40 -0
- data/lib/active_fedora/indexing/map.rb +11 -3
- data/lib/active_fedora/indexing/suffix.rb +81 -0
- data/lib/active_fedora/indexing.rb +39 -15
- data/lib/active_fedora/indexing_service.rb +8 -3
- data/lib/active_fedora/inheritance.rb +1 -1
- data/lib/active_fedora/initializing_connection.rb +1 -1
- data/lib/active_fedora/ldp_resource.rb +1 -2
- data/lib/active_fedora/model_classifier.rb +1 -1
- data/lib/active_fedora/null_logger.rb +13 -0
- data/lib/active_fedora/orders/ordered_list.rb +1 -1
- data/lib/active_fedora/persistence.rb +16 -3
- data/lib/active_fedora/property.rb +1 -2
- data/lib/active_fedora/railtie.rb +6 -2
- data/lib/active_fedora/rake_support.rb +1 -1
- data/lib/active_fedora/rdf/field_map.rb +2 -2
- data/lib/active_fedora/rdf/field_map_entry.rb +15 -7
- data/lib/active_fedora/rdf/indexing_service.rb +15 -13
- data/lib/active_fedora/rdf/value_caster.rb +15 -0
- data/lib/active_fedora/rdf.rb +1 -0
- data/lib/active_fedora/relation/finder_methods.rb +28 -40
- data/lib/active_fedora/solr_hit.rb +1 -1
- data/lib/active_fedora/solr_service.rb +20 -2
- data/lib/active_fedora/type/value.rb +2 -4
- data/lib/active_fedora/version.rb +1 -1
- data/lib/active_fedora/with_metadata/metadata_node.rb +4 -1
- data/lib/active_fedora.rb +23 -9
- data/lib/generators/active_fedora/config/fedora/templates/.fcrepo_wrapper +1 -1
- data/lib/generators/active_fedora/config/solr/templates/solr/config/schema.xml +0 -6
- data/lib/generators/active_fedora/config/solr/templates/solr.yml +3 -3
- data/spec/integration/associations_spec.rb +8 -8
- data/spec/integration/attached_files_spec.rb +2 -3
- data/spec/integration/base_spec.rb +43 -39
- data/spec/integration/basic_contains_association_spec.rb +16 -3
- data/spec/integration/collection_association_spec.rb +1 -1
- data/spec/integration/direct_container_spec.rb +2 -2
- data/spec/integration/fedora_solr_sync_spec.rb +1 -1
- data/spec/integration/file_spec.rb +46 -13
- data/spec/integration/generators/fedora_generator_spec.rb +1 -1
- data/spec/integration/generators/solr_generator_spec.rb +1 -1
- data/spec/integration/has_and_belongs_to_many_associations_spec.rb +2 -2
- data/spec/integration/has_many_associations_spec.rb +4 -5
- data/spec/integration/indexing/descendant_fetcher_spec.rb +64 -0
- data/spec/integration/indexing_spec.rb +22 -2
- data/spec/integration/indirect_container_spec.rb +2 -2
- data/spec/integration/query_result_builder_spec.rb +2 -2
- data/spec/integration/scoping_spec.rb +20 -14
- data/spec/integration/solr_hit_spec.rb +1 -1
- data/spec/integration/versionable_spec.rb +2 -2
- data/spec/lib/active_fedora/null_logger_spec.rb +10 -0
- data/spec/unit/active_fedora/indexing/inserter_spec.rb +30 -0
- data/spec/unit/attached_files_spec.rb +36 -8
- data/spec/unit/attributes_spec.rb +8 -14
- data/spec/unit/base_spec.rb +7 -7
- data/spec/unit/builder/has_and_belongs_to_many_spec.rb +1 -1
- data/spec/unit/callback_spec.rb +2 -2
- data/spec/unit/change_set_spec.rb +1 -2
- data/spec/unit/collection_proxy_spec.rb +2 -1
- data/spec/unit/core/fedora_id_translator_spec.rb +2 -2
- data/spec/unit/core/fedora_uri_translator_spec.rb +2 -2
- data/spec/unit/core/logger_spec.rb +18 -0
- data/spec/unit/core_spec.rb +46 -12
- data/spec/unit/default_model_mapper_spec.rb +1 -1
- data/spec/unit/fedora_spec.rb +12 -0
- data/spec/unit/file_configurator_spec.rb +0 -9
- data/spec/unit/file_io_spec.rb +137 -0
- data/spec/unit/file_path_builder_spec.rb +3 -3
- data/spec/unit/file_spec.rb +70 -26
- data/spec/unit/files_hash_spec.rb +1 -2
- data/spec/unit/filter_spec.rb +2 -2
- data/spec/unit/finder_methods_spec.rb +1 -1
- data/spec/unit/fixity_service_spec.rb +65 -5
- data/spec/unit/has_and_belongs_to_many_association_spec.rb +1 -2
- data/spec/unit/has_many_association_spec.rb +2 -2
- data/spec/unit/indexing/map/index_object_spec.rb +25 -0
- data/spec/unit/indexing/map_spec.rb +27 -0
- data/spec/unit/indexing_spec.rb +3 -4
- data/spec/unit/inheritance_spec.rb +28 -1
- data/spec/unit/model_classifier_spec.rb +1 -1
- data/spec/unit/pathing_spec.rb +1 -2
- data/spec/unit/query_result_builder_spec.rb +1 -2
- data/spec/unit/query_spec.rb +4 -4
- data/spec/unit/rdf/indexing_service_spec.rb +18 -4
- data/spec/unit/reflection_spec.rb +2 -2
- data/spec/unit/solr_hit_spec.rb +1 -2
- data/spec/unit/solr_service_spec.rb +52 -4
- data/spec/unit/sparql_insert_spec.rb +1 -1
- data/spec/unit/validations_spec.rb +27 -4
- data/spec/unit/with_metadata/metadata_node_spec.rb +26 -1
- metadata +99 -29
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
module ActiveFedora
|
|
2
|
+
module Indexing
|
|
3
|
+
class DefaultDescriptors
|
|
4
|
+
# The suffix produced depends on the type parameter -- produces suffixes:
|
|
5
|
+
# _tesim - for strings or text fields
|
|
6
|
+
# _dtsim - for dates
|
|
7
|
+
# _isim - for integers
|
|
8
|
+
def self.stored_searchable
|
|
9
|
+
@stored_searchable ||= Descriptor.new(stored_searchable_field_definition, converter: searchable_converter, requires_type: true)
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
# The suffix produced depends on the type parameter -- produces suffixes:
|
|
13
|
+
# _teim - for strings or text fields
|
|
14
|
+
# _dtim - for dates
|
|
15
|
+
# _iim - for integers
|
|
16
|
+
def self.searchable
|
|
17
|
+
@searchable ||= Descriptor.new(searchable_field_definition, converter: searchable_converter, requires_type: true)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# Takes fields which are stored as strings, but we want indexed as dates. (e.g. "November 6th, 2012")
|
|
21
|
+
# produces suffixes:
|
|
22
|
+
# _dtsim - for dates
|
|
23
|
+
def self.dateable
|
|
24
|
+
@dateable ||= Descriptor.new(:date, :stored, :indexed, :multivalued, converter: dateable_converter)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# Produces _sim suffix
|
|
28
|
+
def self.facetable
|
|
29
|
+
@facetable ||= Descriptor.new(:string, :indexed, :multivalued)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# Produces _ssim suffix
|
|
33
|
+
# This is useful for when you only want to match whole words, such as user/group names from the the rightsMetadata datastream
|
|
34
|
+
def self.symbol
|
|
35
|
+
@symbol ||= Descriptor.new(:string, :stored, :indexed, :multivalued)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# The suffix produced depends on the type parameter -- produces suffixes:
|
|
39
|
+
# _tei - for text fields
|
|
40
|
+
# _si - for strings
|
|
41
|
+
# _dti - for dates
|
|
42
|
+
# _ii - for integers
|
|
43
|
+
def self.sortable
|
|
44
|
+
@sortable ||= Descriptor.new(sortable_field_definition, converter: searchable_converter, requires_type: true)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Fields that are both stored and sortable
|
|
48
|
+
# Produces _ssi suffix if field_type is string
|
|
49
|
+
# Produces _dtsi suffix if field_type is date
|
|
50
|
+
def self.stored_sortable
|
|
51
|
+
@stored_sortable ||= Descriptor.new(->(field_type) { [field_type, :stored, :indexed] }, converter: searchable_converter)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Produces _ssm suffix
|
|
55
|
+
def self.displayable
|
|
56
|
+
@displayable ||= Descriptor.new(:string, :stored, :multivalued)
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Produces _tim suffix (used to be _unstem)
|
|
60
|
+
def self.unstemmed_searchable
|
|
61
|
+
@unstemmed_searchable ||= Descriptor.new(:text, :indexed, :multivalued)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def self.simple
|
|
65
|
+
@simple ||= Descriptor.new(->(field_type) { [field_type, :indexed] })
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
class << self
|
|
69
|
+
def searchable_field_definition
|
|
70
|
+
lambda do |type|
|
|
71
|
+
type = :text_en if [:string, :text].include?(type) # for backwards compatibility with old solr schema
|
|
72
|
+
vals = [type, :indexed, :multivalued]
|
|
73
|
+
vals
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def stored_searchable_field_definition
|
|
78
|
+
lambda do |type|
|
|
79
|
+
type = :text_en if [:string, :text].include?(type) # for backwards compatibility with old solr schema
|
|
80
|
+
if type == :boolean
|
|
81
|
+
[type, :indexed, :stored]
|
|
82
|
+
else
|
|
83
|
+
[type, :indexed, :stored, :multivalued]
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def sortable_field_definition
|
|
89
|
+
lambda do |type|
|
|
90
|
+
vals = [type, :indexed]
|
|
91
|
+
vals
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def searchable_converter
|
|
96
|
+
lambda do |type|
|
|
97
|
+
case type
|
|
98
|
+
when :date, :time
|
|
99
|
+
->(val) { iso8601_date(val) }
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def dateable_converter
|
|
105
|
+
lambda do |_type|
|
|
106
|
+
lambda do |val|
|
|
107
|
+
begin
|
|
108
|
+
iso8601_date(Date.parse(val))
|
|
109
|
+
rescue ArgumentError
|
|
110
|
+
nil
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def iso8601_date(value)
|
|
117
|
+
if value.is_a?(Date) || value.is_a?(Time)
|
|
118
|
+
DateTime.parse(value.to_s).to_time.utc.strftime('%Y-%m-%dT%H:%M:%SZ')
|
|
119
|
+
elsif !value.empty?
|
|
120
|
+
DateTime.parse(value).to_time.utc.strftime('%Y-%m-%dT%H:%M:%SZ')
|
|
121
|
+
end
|
|
122
|
+
rescue ArgumentError
|
|
123
|
+
raise ArgumentError, "Unable to parse `#{value}' as a date-time object"
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
end
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
module ActiveFedora
|
|
2
|
+
module Indexing
|
|
3
|
+
# Finds all descendent URIs of a given repo URI (usually the base URI).
|
|
4
|
+
#
|
|
5
|
+
# This is a slow and non-performant thing to do, we need to fetch every single
|
|
6
|
+
# object from the repo.
|
|
7
|
+
#
|
|
8
|
+
# The DescendantFetcher is also capable of partitioning the URIs into "priority" URIs
|
|
9
|
+
# that will be first in the returned list. These prioritized URIs belong to objects
|
|
10
|
+
# with certain hasModel models. This feature is used in some samvera apps that need to
|
|
11
|
+
# index 'permissions' objects before other objects to have the solr indexing work right.
|
|
12
|
+
# And so by default, the prioritized class names are the ones form Hydra::AccessControls,
|
|
13
|
+
# but you can alter the prioritized model name list, or set it to the empty array.
|
|
14
|
+
#
|
|
15
|
+
# DescendantFetcher.new(ActiveFedora.fedora.base_uri).descendent_and_self_uris
|
|
16
|
+
# #=> array including self uri and descendent uris with "prioritized" (by default)
|
|
17
|
+
# Hydra::AccessControls permissions) objects FIRST.
|
|
18
|
+
#
|
|
19
|
+
# Change the default prioritized hasModel names:
|
|
20
|
+
#
|
|
21
|
+
# ActiveFedora::Indexing::DescendantFetcher.default_priority_models = []
|
|
22
|
+
class DescendantFetcher
|
|
23
|
+
HAS_MODEL_PREDICATE = ActiveFedora::RDF::Fcrepo::Model.hasModel
|
|
24
|
+
|
|
25
|
+
class_attribute :default_priority_models, instance_accessor: false
|
|
26
|
+
self.default_priority_models = %w(Hydra::AccessControl Hydra::AccessControl::Permissions).freeze
|
|
27
|
+
|
|
28
|
+
attr_reader :uri, :priority_models
|
|
29
|
+
|
|
30
|
+
def initialize(uri,
|
|
31
|
+
priority_models: self.class.default_priority_models, exclude_self: false)
|
|
32
|
+
@uri = uri
|
|
33
|
+
@priority_models = priority_models
|
|
34
|
+
@exclude_self = exclude_self
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# @return [Array<String>] uris starting with priority models
|
|
38
|
+
def descendant_and_self_uris
|
|
39
|
+
partitioned = descendant_and_self_uris_partitioned
|
|
40
|
+
partitioned[:priority] + partitioned[:other]
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# returns a hash where key :priority is an array of all prioritized
|
|
44
|
+
# type objects, key :other is an array of the rest.
|
|
45
|
+
# @return [Hash<String, Array<String>>] uris sorted into :priority and :other
|
|
46
|
+
def descendant_and_self_uris_partitioned
|
|
47
|
+
model_partitioned = descendant_and_self_uris_partitioned_by_model
|
|
48
|
+
{ priority: model_partitioned.slice(*priority_models).values.flatten,
|
|
49
|
+
other: model_partitioned.slice(*(model_partitioned.keys - priority_models)).values.flatten }
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Returns a hash where keys are model names
|
|
53
|
+
# This is useful if you need to action on certain models and want finer grainularity than priority/other
|
|
54
|
+
# @return [Hash<String, Array<String>>] uris sorted by model names
|
|
55
|
+
def descendant_and_self_uris_partitioned_by_model
|
|
56
|
+
# GET could be slow if it's a big resource, we're using HEAD to avoid this problem,
|
|
57
|
+
# but this causes more requests to Fedora.
|
|
58
|
+
return partitioned_uris unless rdf_resource.head.rdf_source?
|
|
59
|
+
|
|
60
|
+
add_self_to_partitioned_uris unless @exclude_self
|
|
61
|
+
|
|
62
|
+
immediate_descendant_uris = rdf_graph.query(predicate: ::RDF::Vocab::LDP.contains).map { |descendant| descendant.object.to_s }
|
|
63
|
+
immediate_descendant_uris.each do |descendant_uri|
|
|
64
|
+
self.class.new(
|
|
65
|
+
descendant_uri,
|
|
66
|
+
priority_models: priority_models
|
|
67
|
+
).descendant_and_self_uris_partitioned_by_model.tap do |descendant_partitioned|
|
|
68
|
+
descendant_partitioned.keys.each do |k|
|
|
69
|
+
partitioned_uris[k] ||= []
|
|
70
|
+
partitioned_uris[k].concat descendant_partitioned[k]
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
partitioned_uris
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
protected
|
|
78
|
+
|
|
79
|
+
def rdf_resource
|
|
80
|
+
@rdf_resource ||= Ldp::Resource::RdfSource.new(ActiveFedora.fedora.connection, uri)
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def rdf_graph
|
|
84
|
+
@rdf_graph ||= rdf_resource.graph
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def partitioned_uris
|
|
88
|
+
@partitioned_uris ||= {}
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def rdf_graph_models
|
|
92
|
+
rdf_graph.query(predicate: HAS_MODEL_PREDICATE).collect(&:object).collect do |rdf_object|
|
|
93
|
+
rdf_object.to_s if rdf_object.literal?
|
|
94
|
+
end.compact
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def add_self_to_partitioned_uris
|
|
98
|
+
rdf_graph_models.each do |model|
|
|
99
|
+
partitioned_uris[model] ||= []
|
|
100
|
+
partitioned_uris[model] << rdf_resource.subject
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
end
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
module ActiveFedora
|
|
2
|
+
module Indexing
|
|
3
|
+
class Descriptor
|
|
4
|
+
attr_reader :index_type
|
|
5
|
+
def initialize(*args)
|
|
6
|
+
if args.last.is_a? Hash
|
|
7
|
+
opts = args.pop
|
|
8
|
+
@converter = opts[:converter]
|
|
9
|
+
@type_required = opts[:requires_type]
|
|
10
|
+
end
|
|
11
|
+
@index_type = args
|
|
12
|
+
raise Solrizer::InvalidIndexDescriptor, "Invalid index type passed to Sorizer.solr_name. It should be an array like [:string, :indexed, :stored, :multivalued]. You provided: `#{@index_type}'" unless index_type.is_a? Array
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def name_and_converter(field_name, args = nil)
|
|
16
|
+
args ||= {}
|
|
17
|
+
field_type = args[:type]
|
|
18
|
+
if type_required?
|
|
19
|
+
raise ArgumentError, "Must provide a :type argument when index_type is `#{self}' for #{field_name}" unless field_type
|
|
20
|
+
end
|
|
21
|
+
[field_name.to_s + suffix(field_type), converter(field_type)]
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def type_required?
|
|
25
|
+
@type_required
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def evaluate_suffix(field_type)
|
|
29
|
+
Suffix.new(index_type.first.is_a?(Proc) ? index_type.first.call(field_type) : index_type.dup)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
protected
|
|
33
|
+
|
|
34
|
+
# Suffix can be overridden if you want a different method of grabbing the suffix
|
|
35
|
+
def suffix(field_type)
|
|
36
|
+
evaluate_suffix(field_type).to_s
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def converter(field_type)
|
|
40
|
+
@converter.call(field_type) if @converter
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
module ActiveFedora
|
|
2
|
+
module Indexing
|
|
3
|
+
class FieldMapper
|
|
4
|
+
class_attribute :id_field, :descriptors
|
|
5
|
+
# set defaults
|
|
6
|
+
self.id_field = 'id'
|
|
7
|
+
self.descriptors = [DefaultDescriptors]
|
|
8
|
+
|
|
9
|
+
# @api
|
|
10
|
+
# @params [Hash] doc the hash to insert the value into
|
|
11
|
+
# @params [String] name the name of the field (without the suffix)
|
|
12
|
+
# @params [String,Date] value the value to be inserted
|
|
13
|
+
# @params [Array,Hash] indexer_args the arguments that find the indexer
|
|
14
|
+
# @returns [Hash] doc the document that was provided with the new field (replacing any field with the same name)
|
|
15
|
+
def set_field(doc, name, value, *indexer_args)
|
|
16
|
+
# adding defaults indexer
|
|
17
|
+
indexer_args = [:stored_searchable] if indexer_args.empty?
|
|
18
|
+
doc.merge! solr_names_and_values(name, value, indexer_args)
|
|
19
|
+
doc
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# @api
|
|
23
|
+
# Given a field name, index_type, etc., returns the corresponding Solr name.
|
|
24
|
+
# TODO field type is the input format, maybe we could just detect that?
|
|
25
|
+
# @param [String] field_name the ruby (term) name which will get a suffix appended to become a Solr field name
|
|
26
|
+
# @param opts - index_type is only needed if the FieldDescriptor requires it (e.g. :searcahble)
|
|
27
|
+
# @return [String] name of the solr field, based on the params
|
|
28
|
+
def solr_name(field_name, *opts)
|
|
29
|
+
index_type, args = if opts.first.is_a? Hash
|
|
30
|
+
[:stored_searchable, opts.first]
|
|
31
|
+
elsif opts.empty?
|
|
32
|
+
[:stored_searchable, { type: :text }]
|
|
33
|
+
else
|
|
34
|
+
[opts[0], opts[1] || { type: :string }]
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
indexer(index_type).name_and_converter(field_name, args).first
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Given a field name-value pair, a data type, and an array of index types, returns a hash of
|
|
41
|
+
# mapped names and values. The values in the hash are _arrays_, and may contain multiple values.
|
|
42
|
+
def solr_names_and_values(field_name, field_value, index_types)
|
|
43
|
+
return {} if field_value.nil?
|
|
44
|
+
|
|
45
|
+
# Determine the set of index types
|
|
46
|
+
index_types = Array(index_types)
|
|
47
|
+
index_types.uniq!
|
|
48
|
+
index_types.dup.each do |index_type|
|
|
49
|
+
if index_type.to_s =~ /^not_(.*)/
|
|
50
|
+
index_types.delete index_type # not_foo
|
|
51
|
+
index_types.delete Regexp.last_match(1).to_sym # foo
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Map names and values
|
|
56
|
+
|
|
57
|
+
results = {}
|
|
58
|
+
|
|
59
|
+
# Time seems to extend enumerable, so wrap it so we don't interate over each of its elements.
|
|
60
|
+
field_value = [field_value] if field_value.is_a? Time
|
|
61
|
+
|
|
62
|
+
index_types.each do |index_type|
|
|
63
|
+
Array(field_value).each do |single_value|
|
|
64
|
+
# Get mapping for field
|
|
65
|
+
descriptor = indexer(index_type)
|
|
66
|
+
data_type = extract_type(single_value)
|
|
67
|
+
name, converter = descriptor.name_and_converter(field_name, type: data_type)
|
|
68
|
+
next unless name
|
|
69
|
+
|
|
70
|
+
# Is there a custom converter?
|
|
71
|
+
# TODO instead of a custom converter, look for input data type and output data type. Create a few methods that can do that cast.
|
|
72
|
+
|
|
73
|
+
value = if converter
|
|
74
|
+
if converter.arity == 1
|
|
75
|
+
converter.call(single_value)
|
|
76
|
+
else
|
|
77
|
+
converter.call(single_value, field_name)
|
|
78
|
+
end
|
|
79
|
+
elsif data_type == :boolean
|
|
80
|
+
single_value
|
|
81
|
+
else
|
|
82
|
+
single_value.to_s
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# Add mapped name & value, unless it's a duplicate
|
|
86
|
+
if descriptor.evaluate_suffix(data_type).multivalued?
|
|
87
|
+
values = (results[name] ||= [])
|
|
88
|
+
values << value unless value.nil? || values.include?(value)
|
|
89
|
+
else
|
|
90
|
+
Solrizer.logger.warn "Setting #{name} to `#{value}', but it already had `#{results[name]}'" if results[name] && Solrizer.logger
|
|
91
|
+
results[name] = value
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
results
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
private
|
|
100
|
+
|
|
101
|
+
# @param [Symbol, String, Descriptor] index_type is a Descriptor, a symbol that references a method that returns a Descriptor, or a string which will be used as the suffix.
|
|
102
|
+
# @return [Descriptor]
|
|
103
|
+
def indexer(index_type)
|
|
104
|
+
index_type = case index_type
|
|
105
|
+
when Symbol
|
|
106
|
+
index_type_macro(index_type)
|
|
107
|
+
when String
|
|
108
|
+
StringDescriptor.new(index_type)
|
|
109
|
+
when Descriptor
|
|
110
|
+
index_type
|
|
111
|
+
else
|
|
112
|
+
raise Solrizer::InvalidIndexDescriptor, "#{index_type.class} is not a valid indexer_type. Use a String, Symbol or Descriptor."
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
raise InvalidIndexDescriptor, "index type should be an Descriptor, you passed: #{index_type.class}" unless index_type.is_a? Descriptor
|
|
116
|
+
index_type
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# @param index_type [Symbol]
|
|
120
|
+
# search through the descriptors (class attribute) until a module is found that responds to index_type, then call it.
|
|
121
|
+
def index_type_macro(index_type)
|
|
122
|
+
klass = self.class.descriptors.find { |descriptor_klass| descriptor_klass.respond_to? index_type }
|
|
123
|
+
if klass
|
|
124
|
+
klass.send(index_type)
|
|
125
|
+
else
|
|
126
|
+
raise UnknownIndexMacro, "Unable to find `#{index_type}' in #{self.class.descriptors}"
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def extract_type(value)
|
|
131
|
+
case value
|
|
132
|
+
when NilClass
|
|
133
|
+
nil
|
|
134
|
+
when Integer # In ruby < 2.4, Fixnum extends Integer
|
|
135
|
+
:integer
|
|
136
|
+
when DateTime
|
|
137
|
+
:time
|
|
138
|
+
when TrueClass, FalseClass
|
|
139
|
+
:boolean
|
|
140
|
+
else
|
|
141
|
+
value.class.to_s.underscore.to_sym
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
end
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
module ActiveFedora
|
|
2
|
+
module Indexing
|
|
3
|
+
# Utilities for adding fields to index documents
|
|
4
|
+
class Inserter
|
|
5
|
+
# @param [String] field_name_base the field name
|
|
6
|
+
# @param [String] value the value to insert into the index
|
|
7
|
+
# @param [Array<Symbol>] index_as the index type suffixes
|
|
8
|
+
# @param [Hash] solr_doc the index doc to add to
|
|
9
|
+
# @example:
|
|
10
|
+
# solr_doc = {}
|
|
11
|
+
# create_and_insert_terms('title', 'War and Peace', [:displayable, :searchable], solr_doc)
|
|
12
|
+
# solr_doc
|
|
13
|
+
# # => {"title_ssm"=>["War and Peace"], "title_teim"=>["War and Peace"]}
|
|
14
|
+
def self.create_and_insert_terms(field_name_base, value, index_as, solr_doc)
|
|
15
|
+
index_as.each do |indexer|
|
|
16
|
+
insert_field(solr_doc, field_name_base, value, indexer)
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# @params [Hash] doc the hash to insert the value into
|
|
21
|
+
# @params [String] name the name of the field (without the suffix)
|
|
22
|
+
# @params [String,Date,Array] value the value (or array of values) to be inserted
|
|
23
|
+
# @params [Array,Hash] indexer_args the arguments that find the indexer
|
|
24
|
+
# @returns [Hash] doc the document that was provided with the new field inserted
|
|
25
|
+
def self.insert_field(doc, name, value, *indexer_args)
|
|
26
|
+
# adding defaults indexer
|
|
27
|
+
indexer_args = [:stored_searchable] if indexer_args.empty?
|
|
28
|
+
ActiveFedora.index_field_mapper.solr_names_and_values(name, value, indexer_args).each do |k, v|
|
|
29
|
+
doc[k] ||= []
|
|
30
|
+
if v.is_a? Array
|
|
31
|
+
doc[k] += v
|
|
32
|
+
else
|
|
33
|
+
doc[k] = v
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
doc
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
@@ -11,7 +11,15 @@ module ActiveFedora::Indexing
|
|
|
11
11
|
end
|
|
12
12
|
|
|
13
13
|
def dup
|
|
14
|
-
self.class.new(
|
|
14
|
+
self.class.new(to_hash)
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def merge(new_hash)
|
|
18
|
+
self.class.new(to_hash.merge(new_hash))
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def to_hash
|
|
22
|
+
@hash.deep_dup
|
|
15
23
|
end
|
|
16
24
|
|
|
17
25
|
# this enables a cleaner API for solr integration
|
|
@@ -19,8 +27,8 @@ module ActiveFedora::Indexing
|
|
|
19
27
|
attr_accessor :data_type, :behaviors, :term
|
|
20
28
|
attr_reader :key
|
|
21
29
|
|
|
22
|
-
def initialize(name, &_block)
|
|
23
|
-
@behaviors =
|
|
30
|
+
def initialize(name, behaviors: [], &_block)
|
|
31
|
+
@behaviors = behaviors
|
|
24
32
|
@data_type = :string
|
|
25
33
|
@key = name
|
|
26
34
|
yield self if block_given?
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
require 'ostruct'
|
|
2
|
+
|
|
3
|
+
module ActiveFedora
|
|
4
|
+
module Indexing
|
|
5
|
+
class Suffix
|
|
6
|
+
def initialize(*fields)
|
|
7
|
+
@fields = fields.flatten
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def multivalued?
|
|
11
|
+
has_field? :multivalued
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def stored?
|
|
15
|
+
has_field? :stored
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def indexed?
|
|
19
|
+
has_field? :indexed
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def has_field?(f)
|
|
23
|
+
(f.to_sym == :type) || @fields.include?(f.to_sym)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def data_type
|
|
27
|
+
@fields.first
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def to_s
|
|
31
|
+
raise Solrizer::InvalidIndexDescriptor, "Missing datatype for #{@fields}" unless data_type
|
|
32
|
+
|
|
33
|
+
field_suffix = [config.suffix_delimiter]
|
|
34
|
+
|
|
35
|
+
config.fields.select { |f| has_field? f }.each do |f|
|
|
36
|
+
key = :"#{f}_suffix"
|
|
37
|
+
field_suffix << if config.send(key).is_a? Proc
|
|
38
|
+
config.send(key).call(@fields)
|
|
39
|
+
else
|
|
40
|
+
config.send(key)
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
field_suffix.join
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def self.config
|
|
48
|
+
@config ||= OpenStruct.new fields: [:type, :stored, :indexed, :multivalued],
|
|
49
|
+
suffix_delimiter: '_',
|
|
50
|
+
type_suffix: (lambda do |fields|
|
|
51
|
+
type = fields.first
|
|
52
|
+
case type
|
|
53
|
+
when :string, :symbol # TODO: `:symbol' usage ought to be deprecated
|
|
54
|
+
's'
|
|
55
|
+
when :text
|
|
56
|
+
't'
|
|
57
|
+
when :text_en
|
|
58
|
+
'te'
|
|
59
|
+
when :date, :time
|
|
60
|
+
'dt'
|
|
61
|
+
when :integer
|
|
62
|
+
'i'
|
|
63
|
+
when :boolean
|
|
64
|
+
'b'
|
|
65
|
+
when :long
|
|
66
|
+
'lt'
|
|
67
|
+
else
|
|
68
|
+
raise Solrizer::InvalidIndexDescriptor, "Invalid datatype `#{type.inspect}'. Must be one of: :date, :time, :text, :text_en, :string, :symbol, :integer, :boolean"
|
|
69
|
+
end
|
|
70
|
+
end),
|
|
71
|
+
stored_suffix: 's',
|
|
72
|
+
indexed_suffix: 'i',
|
|
73
|
+
multivalued_suffix: 'm'
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def config
|
|
77
|
+
@config ||= self.class.config.dup
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
@@ -12,7 +12,13 @@ module ActiveFedora
|
|
|
12
12
|
extend ActiveSupport::Autoload
|
|
13
13
|
|
|
14
14
|
eager_autoload do
|
|
15
|
+
autoload :DefaultDescriptors
|
|
16
|
+
autoload :Descriptor
|
|
17
|
+
autoload :DescendantFetcher
|
|
18
|
+
autoload :FieldMapper
|
|
19
|
+
autoload :Inserter
|
|
15
20
|
autoload :Map
|
|
21
|
+
autoload :Suffix
|
|
16
22
|
end
|
|
17
23
|
|
|
18
24
|
included do
|
|
@@ -82,26 +88,44 @@ module ActiveFedora
|
|
|
82
88
|
end
|
|
83
89
|
end
|
|
84
90
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
91
|
+
# @param [Integer] batch_size - The number of Fedora objects to process for each SolrService.add call. Default 50.
|
|
92
|
+
# @param [Boolean] softCommit - Do we perform a softCommit when we add the to_solr objects to SolrService. Default true.
|
|
93
|
+
# @param [Boolean] progress_bar - If true output progress bar information. Default false.
|
|
94
|
+
# @param [Boolean] final_commit - If true perform a hard commit to the Solr service at the completion of the batch of updates. Default false.
|
|
95
|
+
def reindex_everything(batch_size: 50, softCommit: true, progress_bar: false, final_commit: false)
|
|
96
|
+
# skip root url
|
|
97
|
+
descendants = descendant_uris(ActiveFedora.fedora.base_uri, exclude_uri: true)
|
|
98
|
+
|
|
99
|
+
batch = []
|
|
100
|
+
|
|
101
|
+
progress_bar_controller = ProgressBar.create(total: descendants.count, format: "%t: |%B| %p%% %e") if progress_bar
|
|
102
|
+
|
|
88
103
|
descendants.each do |uri|
|
|
89
104
|
logger.debug "Re-index everything ... #{uri}"
|
|
90
|
-
|
|
105
|
+
|
|
106
|
+
batch << ActiveFedora::Base.find(ActiveFedora::Base.uri_to_id(uri)).to_solr
|
|
107
|
+
|
|
108
|
+
if (batch.count % batch_size).zero?
|
|
109
|
+
SolrService.add(batch, softCommit: softCommit)
|
|
110
|
+
batch.clear
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
progress_bar_controller.increment if progress_bar_controller
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
if batch.present?
|
|
117
|
+
SolrService.add(batch, softCommit: softCommit)
|
|
118
|
+
batch.clear
|
|
91
119
|
end
|
|
92
|
-
end
|
|
93
120
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
# but this causes more requests to Fedora.
|
|
98
|
-
return [] unless resource.head.rdf_source?
|
|
99
|
-
immediate_descendant_uris = resource.graph.query(predicate: ::RDF::Vocab::LDP.contains).map { |descendant| descendant.object.to_s }
|
|
100
|
-
all_descendants_uris = [uri]
|
|
101
|
-
immediate_descendant_uris.each do |descendant_uri|
|
|
102
|
-
all_descendants_uris += descendant_uris(descendant_uri)
|
|
121
|
+
if final_commit
|
|
122
|
+
logger.debug "Solr hard commit..."
|
|
123
|
+
SolrService.commit
|
|
103
124
|
end
|
|
104
|
-
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def descendant_uris(uri, exclude_uri: false)
|
|
128
|
+
DescendantFetcher.new(uri, exclude_self: exclude_uri).descendant_and_self_uris
|
|
105
129
|
end
|
|
106
130
|
end
|
|
107
131
|
end
|
|
@@ -32,8 +32,8 @@ module ActiveFedora
|
|
|
32
32
|
# @return [Hash] the solr document
|
|
33
33
|
def generate_solr_document
|
|
34
34
|
solr_doc = {}
|
|
35
|
-
|
|
36
|
-
|
|
35
|
+
ActiveFedora.index_field_mapper.set_field(solr_doc, 'system_create', c_time, :stored_sortable)
|
|
36
|
+
ActiveFedora.index_field_mapper.set_field(solr_doc, 'system_modified', m_time, :stored_sortable)
|
|
37
37
|
solr_doc[QueryResultBuilder::HAS_MODEL_SOLR_FIELD] = object.has_model
|
|
38
38
|
solr_doc[ActiveFedora.id_field.to_sym] = object.id
|
|
39
39
|
object.declared_attached_files.each do |name, file|
|
|
@@ -61,7 +61,12 @@ module ActiveFedora
|
|
|
61
61
|
# Serialize the resource's RDF relationships to solr
|
|
62
62
|
# @param [Hash] solr_doc @deafult an empty Hash
|
|
63
63
|
def solrize_rdf_assertions(solr_doc = {})
|
|
64
|
-
solr_doc.merge
|
|
64
|
+
solr_doc.merge rdf_indexer.generate_solr_document
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# @return IndexingService
|
|
68
|
+
def rdf_indexer
|
|
69
|
+
rdf_service.new(object, object.class.index_config)
|
|
65
70
|
end
|
|
66
71
|
end
|
|
67
72
|
end
|
|
@@ -12,7 +12,7 @@ module ActiveFedora
|
|
|
12
12
|
# If B < A and C < B and if A is an abstract_class then both B.base_class
|
|
13
13
|
# and C.base_class would return B as the answer since A is an abstract_class.
|
|
14
14
|
def base_class
|
|
15
|
-
return File if self
|
|
15
|
+
return File if self <= File
|
|
16
16
|
|
|
17
17
|
unless self <= Base
|
|
18
18
|
raise ActiveFedoraError, "#{name} doesn't belong in a hierarchy descending from ActiveFedora"
|