cdmdexer 0.17.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +10 -0
- data/.rubocop.yml +4 -0
- data/.travis.yml +8 -0
- data/CODE_OF_CONDUCT.md +49 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +21 -0
- data/README.md +149 -0
- data/Rakefile +11 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/cdmdexer.gemspec +35 -0
- data/lib/cdmdexer/cdm_item.rb +89 -0
- data/lib/cdmdexer/default_cdm_notification.rb +8 -0
- data/lib/cdmdexer/default_completed_callback.rb +8 -0
- data/lib/cdmdexer/default_loader_notification.rb +8 -0
- data/lib/cdmdexer/default_oai_notification.rb +8 -0
- data/lib/cdmdexer/default_solr.rb +35 -0
- data/lib/cdmdexer/etl_by_set_specs.rb +18 -0
- data/lib/cdmdexer/etl_worker.rb +113 -0
- data/lib/cdmdexer/field_formatter.rb +13 -0
- data/lib/cdmdexer/field_mapping.rb +28 -0
- data/lib/cdmdexer/field_transformer.rb +41 -0
- data/lib/cdmdexer/filtered_set_specs.rb +41 -0
- data/lib/cdmdexer/formatters.rb +169 -0
- data/lib/cdmdexer/hooks.rb +31 -0
- data/lib/cdmdexer/load_worker.rb +36 -0
- data/lib/cdmdexer/loader.rb +19 -0
- data/lib/cdmdexer/oai_client.rb +26 -0
- data/lib/cdmdexer/oai_request.rb +100 -0
- data/lib/cdmdexer/rake_task.rb +6 -0
- data/lib/cdmdexer/record_transformer.rb +25 -0
- data/lib/cdmdexer/regex_filter_callback.rb +19 -0
- data/lib/cdmdexer/tasks/delete.rake +12 -0
- data/lib/cdmdexer/tasks/etl.rake +96 -0
- data/lib/cdmdexer/transform_worker.rb +93 -0
- data/lib/cdmdexer/transformer.rb +171 -0
- data/lib/cdmdexer/version.rb +3 -0
- data/lib/cdmdexer.rb +26 -0
- data/travis.yml +6 -0
- metadata +223 -0
@@ -0,0 +1,93 @@
|
|
1
|
+
require 'sidekiq'
|
2
|
+
module CDMDEXER
|
3
|
+
class TransformWorker
|
4
|
+
include Sidekiq::Worker
|
5
|
+
attr_reader :records,
|
6
|
+
:solr_config,
|
7
|
+
:cdm_endpoint,
|
8
|
+
:oai_endpoint,
|
9
|
+
:field_mappings,
|
10
|
+
:batch_size
|
11
|
+
|
12
|
+
attr_writer :cdm_item_klass,
|
13
|
+
:load_worker_klass,
|
14
|
+
:transformer_klass,
|
15
|
+
:transformer_worker_klass
|
16
|
+
|
17
|
+
def perform(records,
|
18
|
+
solr_config,
|
19
|
+
cdm_endpoint,
|
20
|
+
oai_endpoint,
|
21
|
+
field_mappings,
|
22
|
+
batch_size)
|
23
|
+
|
24
|
+
@records = records
|
25
|
+
@solr_config = solr_config
|
26
|
+
@cdm_endpoint = cdm_endpoint
|
27
|
+
@oai_endpoint = oai_endpoint
|
28
|
+
@field_mappings = field_mappings
|
29
|
+
@batch_size = batch_size
|
30
|
+
transform_and_load!
|
31
|
+
transform_and_load_compounds!
|
32
|
+
end
|
33
|
+
|
34
|
+
def cdm_item_klass
|
35
|
+
@cdm_item_klass ||= CdmItem
|
36
|
+
end
|
37
|
+
|
38
|
+
def transformer_klass
|
39
|
+
@transformer_klass ||= Transformer
|
40
|
+
end
|
41
|
+
|
42
|
+
def load_worker_klass
|
43
|
+
@load_worker_klass ||= LoadWorker
|
44
|
+
end
|
45
|
+
|
46
|
+
def transformer_worker_klass
|
47
|
+
@transformer_worker_klass ||= TransformWorker
|
48
|
+
end
|
49
|
+
|
50
|
+
private
|
51
|
+
|
52
|
+
# Recursivly call the transformer_worker with all the the compound
|
53
|
+
# data we have collected in the first pass
|
54
|
+
def transform_and_load_compounds!
|
55
|
+
compound_records.each_slice(batch_size) do |compound_records_batch|
|
56
|
+
transformer_worker_klass.perform_async(
|
57
|
+
compound_records_batch,
|
58
|
+
solr_config,
|
59
|
+
cdm_endpoint,
|
60
|
+
oai_endpoint,
|
61
|
+
field_mappings,
|
62
|
+
batch_size
|
63
|
+
)
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
def transform_and_load!
|
68
|
+
load_worker_klass.perform_async(transformed_records, [], solr_config)
|
69
|
+
end
|
70
|
+
|
71
|
+
def compound_records
|
72
|
+
cmd_items.map(&:page).flatten
|
73
|
+
end
|
74
|
+
|
75
|
+
def cmd_items
|
76
|
+
@cdm_items ||= records.map do |record|
|
77
|
+
cdm_item_klass.new(record: record, cdm_endpoint: cdm_endpoint)
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
def transformed_records
|
82
|
+
transformer_klass.new(cdm_records: cmd_items.map(&:to_h),
|
83
|
+
oai_endpoint: oai_endpoint,
|
84
|
+
field_mappings: field_mappings).records
|
85
|
+
end
|
86
|
+
|
87
|
+
def complete_records
|
88
|
+
records.map do |record|
|
89
|
+
cdm_request(*identifier)
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
@@ -0,0 +1,171 @@
|
|
1
|
+
require 'json'
|
2
|
+
require 'titleize'
|
3
|
+
|
4
|
+
module CDMDEXER
|
5
|
+
class Transformer
|
6
|
+
attr_reader :cdm_records,
|
7
|
+
:oai_endpoint,
|
8
|
+
:field_mappings,
|
9
|
+
:record_transformer,
|
10
|
+
:cache_klass,
|
11
|
+
:oai_request_klass
|
12
|
+
|
13
|
+
def initialize(cdm_records: [],
|
14
|
+
oai_endpoint: :MISSING_OAI_ENDPOINT,
|
15
|
+
field_mappings: false,
|
16
|
+
record_transformer: RecordTransformer,
|
17
|
+
cache_klass: ::Rails,
|
18
|
+
oai_request_klass: OaiRequest)
|
19
|
+
@cdm_records = cdm_records
|
20
|
+
@oai_endpoint = oai_endpoint
|
21
|
+
@field_mappings = field_mappings ? field_mappings : default_field_mappings
|
22
|
+
@record_transformer = record_transformer
|
23
|
+
@cache_klass = cache_klass
|
24
|
+
@oai_request_klass = oai_request_klass
|
25
|
+
end
|
26
|
+
|
27
|
+
def records
|
28
|
+
cdm_records.map { |record| to_solr(record) }.compact
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def sets
|
34
|
+
@oai_request ||=
|
35
|
+
cache_klass.cache.fetch("cdmdexer_sets", expires_in: 10.minutes) do
|
36
|
+
oai_request_klass.new(endpoint_url: oai_endpoint).set_lookup
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def mappings
|
41
|
+
field_mappings.map { |config| FieldMapping.new(config: config) }
|
42
|
+
end
|
43
|
+
|
44
|
+
def to_solr(record)
|
45
|
+
# Remove empty records (move this behavior to the CONTENTdm API gem) and
|
46
|
+
# bail early on the transformation process
|
47
|
+
if {'id' => record['id']} == record
|
48
|
+
return nil
|
49
|
+
else
|
50
|
+
record_transformer.new(record: record.merge('oai_sets' => sets),
|
51
|
+
field_mappings: mappings).transform!
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def default_field_mappings
|
56
|
+
[
|
57
|
+
{dest_path: 'location_llsi', origin_path: '/', formatters: [LocationFormatter]},
|
58
|
+
{dest_path: 'id', origin_path: 'id', formatters: [StripFormatter]},
|
59
|
+
{dest_path: 'setspec_ssi', origin_path: '/', formatters: [AddSetSpecFormatter, SetSpecFormatter]},
|
60
|
+
{dest_path: 'collection_name_ssi', origin_path: '/', formatters: [AddSetSpecFormatter, CollectionNameFormatter]},
|
61
|
+
{dest_path: 'collection_name_tei', origin_path: '/', formatters: [AddSetSpecFormatter, CollectionNameFormatter]},
|
62
|
+
{dest_path: 'collection_description_tesi', origin_path: '/', formatters: [AddSetSpecFormatter, CollectionDescriptionFormatter, FilterBadCollections]},
|
63
|
+
{dest_path: 'parent_collection_name_ssi', origin_path: 'par', formatters: [StripFormatter]},
|
64
|
+
{dest_path: 'parent_collection_name_tei', origin_path: 'par', formatters: [StripFormatter]},
|
65
|
+
{dest_path: 'parent_collection_description_tei', origin_path: 'par', formatters: [StripFormatter]},
|
66
|
+
{dest_path: 'title_tesi', origin_path: 'title', formatters: [StripFormatter]},
|
67
|
+
{dest_path: 'title_ssi', origin_path: 'title', formatters: [StripFormatter]},
|
68
|
+
{dest_path: 'title_sort', origin_path: 'title', formatters: [StripFormatter]},
|
69
|
+
{dest_path: 'title_unstem_search', origin_path: 'title', formatters: [StripFormatter]},
|
70
|
+
{dest_path: 'contributor_teim', origin_path: 'contri', formatters: [StripFormatter]},
|
71
|
+
{dest_path: 'contributor_unstem_search', origin_path: 'contri', formatters: [StripFormatter]},
|
72
|
+
{dest_path: 'contributor_ssim', origin_path: 'contri', formatters: [SplitFormatter, StripFormatter]},
|
73
|
+
{dest_path: 'creator_tesi', origin_path: 'photog', formatters: [JoinFormatter, StripFormatter]},
|
74
|
+
{dest_path: 'creator_unstem_search', origin_path: 'photog', formatters: [StripFormatter]},
|
75
|
+
{dest_path: 'creator_ssim', origin_path: 'photog', formatters: [SplitFormatter, StripFormatter]},
|
76
|
+
{dest_path: 'creator_sort', origin_path: 'photog', formatters: [StripFormatter]},
|
77
|
+
{dest_path: 'description_tei', origin_path: 'descri', formatters: [StripFormatter]},
|
78
|
+
{dest_path: 'description_ts', origin_path: 'descri', formatters: [StripFormatter]},
|
79
|
+
{dest_path: 'dat_ssi', origin_path: 'dat', formatters: [StripFormatter]},
|
80
|
+
{dest_path: 'dat_tesi', origin_path: 'dat', formatters: [StripFormatter]},
|
81
|
+
{dest_path: 'dat_sort', origin_path: 'dat', formatters: [StripFormatter]},
|
82
|
+
{dest_path: 'publishing_agency_tei', origin_path: 'publia', formatters: [StripFormatter]},
|
83
|
+
{dest_path: 'publishing_agency_unstem_search', origin_path: 'publia', formatters: [StripFormatter]},
|
84
|
+
{dest_path: 'publishing_agency_ssi', origin_path: 'publia', formatters: [StripFormatter]},
|
85
|
+
{dest_path: 'dimensions_ssi', origin_path: 'dimens', formatters: [StripFormatter]},
|
86
|
+
{dest_path: 'topic_teim', origin_path: 'genera', formatters: [StripFormatter, SplitFormatter, StripFormatter]},
|
87
|
+
{dest_path: 'topic_ssim', origin_path: 'genera', formatters: [Titlieze, StripFormatter, SplitFormatter, StripFormatter]},
|
88
|
+
{dest_path: 'topic_unstem_search', origin_path: 'genera', formatters: [StripSemicolonFormatter, StripFormatter]},
|
89
|
+
{dest_path: 'type_ssi', origin_path: 'type', formatters: [Titlieze, StripSemicolonFormatter, StripFormatter]},
|
90
|
+
{dest_path: 'type_tesi', origin_path: 'type', formatters: [Titlieze, StripSemicolonFormatter, StripFormatter]},
|
91
|
+
{dest_path: 'physical_format_ssi', origin_path: 'physic', formatters: [StripSemicolonFormatter]},
|
92
|
+
{dest_path: 'physical_format_tesi', origin_path: 'physic', formatters: [StripSemicolonFormatter]},
|
93
|
+
{dest_path: 'formal_subject_unstem_search', origin_path: 'specif', formatters: [StripFormatter]},
|
94
|
+
{dest_path: 'formal_subject_ssim', origin_path: 'specif', formatters: [Titlieze, StripFormatter, SplitFormatter, StripFormatter]},
|
95
|
+
{dest_path: 'formal_subject_teim', origin_path: 'specif', formatters: [Titlieze, StripFormatter, SplitFormatter, StripFormatter]},
|
96
|
+
{dest_path: 'subject_unstem_search', origin_path: 'subjec', formatters: [StripFormatter]},
|
97
|
+
{dest_path: 'subject_teim', origin_path: 'subjec', formatters: [StripFormatter, SplitFormatter, StripFormatter]},
|
98
|
+
{dest_path: 'subject_ssim', origin_path: 'subjec', formatters: [StripFormatter, SplitFormatter, StripFormatter]},
|
99
|
+
{dest_path: 'keyword_unstem_search', origin_path: '/', formatters: [KeywordFormatter, Titlieze, UniqueFormatter, StripFormatter]},
|
100
|
+
{dest_path: 'keyword_tesi', origin_path: '/', formatters: [KeywordFormatter, Titlieze, UniqueFormatter, JoinFormatter, StripFormatter]},
|
101
|
+
{dest_path: 'keyword_ssim', origin_path: '/', formatters: [KeywordFormatter, Titlieze, UniqueFormatter, StripFormatter]},
|
102
|
+
{dest_path: 'city_ssim', origin_path: 'city', formatters: [StripFormatter, SplitFormatter, StripFormatter]},
|
103
|
+
{dest_path: 'city_unstem_search', origin_path: 'city', formatters: [StripFormatter]},
|
104
|
+
{dest_path: 'district_ssi', origin_path: 'distri', formatters: [StripFormatter]},
|
105
|
+
{dest_path: 'district_unstem_search', origin_path: 'distri', formatters: [StripFormatter]},
|
106
|
+
{dest_path: 'county_ssim', origin_path: 'county', formatters: [Titlieze, StripFormatter, SplitFormatter, StripFormatter]},
|
107
|
+
{dest_path: 'county_unstem_search', origin_path: 'county', formatters: [StripFormatter]},
|
108
|
+
{dest_path: 'state_ssi', origin_path: 'state', formatters: [StripFormatter]},
|
109
|
+
{dest_path: 'state_unstem_search', origin_path: 'state', formatters: [StripFormatter]},
|
110
|
+
{dest_path: 'country_ssi', origin_path: 'countr', formatters: [StripFormatter]},
|
111
|
+
{dest_path: 'country_unstem_search', origin_path: 'countr', formatters: [StripFormatter]},
|
112
|
+
{dest_path: 'language_ssi', origin_path: 'langua', formatters: [StripFormatter]},
|
113
|
+
{dest_path: 'language_unstem_search', origin_path: 'langua', formatters: [StripFormatter]},
|
114
|
+
{dest_path: 'contributing_unstem_search', origin_path: 'contra', formatters: [StripFormatter]},
|
115
|
+
{dest_path: 'contributing_organization_tesi', origin_path: 'contra', formatters: [StripFormatter]},
|
116
|
+
{dest_path: 'contributing_organization_ssi', origin_path: 'contra', formatters: [Titlieze, StripFormatter]},
|
117
|
+
{dest_path: 'contact_information_ssi', origin_path: 'contac', formatters: [StripFormatter]},
|
118
|
+
{dest_path: 'rights_ssi', origin_path: 'righta', formatters: [StripFormatter]},
|
119
|
+
{dest_path: 'local_identifier_ssi', origin_path: 'identi', formatters: [StripFormatter]},
|
120
|
+
{dest_path: 'identifier_ssi', origin_path: 'resour', formatters: [StripFormatter]},
|
121
|
+
{dest_path: 'project_ssi', origin_path: 'projec', formatters: [StripFormatter]},
|
122
|
+
{dest_path: 'fiscal_sponsor_ssi', origin_path: 'fiscal', formatters: [StripFormatter]},
|
123
|
+
{dest_path: 'publisher_ssi', origin_path: 'publis', formatters: [StripFormatter]},
|
124
|
+
{dest_path: 'date_ssi', origin_path: 'date', formatters: [StripFormatter]},
|
125
|
+
{dest_path: 'format_tesi', origin_path: 'format', formatters: [StripFormatter]},
|
126
|
+
{dest_path: 'digspa_ssi', origin_path: 'digspa'},
|
127
|
+
{dest_path: 'digspb_ssi', origin_path: 'digspb'},
|
128
|
+
{dest_path: 'digspc_ssi', origin_path: 'digspc'},
|
129
|
+
{dest_path: 'digspd_ssi', origin_path: 'digspd'},
|
130
|
+
{dest_path: 'digspe_ssi', origin_path: 'digspe'},
|
131
|
+
{dest_path: 'digspf_ssi', origin_path: 'digspf'},
|
132
|
+
{dest_path: 'digspg_ssi', origin_path: 'digspg'},
|
133
|
+
{dest_path: 'digsph_ssi', origin_path: 'digsph'},
|
134
|
+
{dest_path: 'digspi_ssi', origin_path: 'digspi'},
|
135
|
+
{dest_path: 'digspj_ssi', origin_path: 'digspj'},
|
136
|
+
{dest_path: 'digspk_ssi', origin_path: 'digspk'},
|
137
|
+
{dest_path: 'transcription_tesi', origin_path: 'transc', formatters: [StripFormatter]},
|
138
|
+
{dest_path: 'translation_tesi', origin_path: 'transl', formatters: [StripFormatter]},
|
139
|
+
{dest_path: 'fullrs_tesi', origin_path: 'fullrs', formatters: [StripFormatter]},
|
140
|
+
{dest_path: 'find_ssi', origin_path: 'find', formatters: [StripFormatter]},
|
141
|
+
{dest_path: 'dmaccess_ssi', origin_path: 'dmaccess', formatters: [StripFormatter]},
|
142
|
+
{dest_path: 'dmimage_ssi', origin_path: 'dmimage', formatters: [StripFormatter]},
|
143
|
+
{dest_path: 'dmcreated_ssi', origin_path: 'dmcreated', formatters: [StripFormatter]},
|
144
|
+
{dest_path: 'dmmodified_ssi', origin_path: 'dmmodified', formatters: [StripFormatter]},
|
145
|
+
{dest_path: 'dmoclcno_ssi', origin_path: 'dmoclcno', formatters: [StripFormatter]},
|
146
|
+
{dest_path: 'restriction_code_ssi', origin_path: 'restrictionCode', formatters: [StripFormatter]},
|
147
|
+
{dest_path: 'cdmfilesize_ssi', origin_path: 'cdmfilesize', formatters: [StripFormatter]},
|
148
|
+
{dest_path: 'cdmfilesizeformatted_ssi', origin_path: 'cdmfilesizeformatted', formatters: [StripFormatter]},
|
149
|
+
{dest_path: 'cdmprintpdf_is', origin_path: 'cdmprintpdf', formatters: [ToIFormatter]},
|
150
|
+
{dest_path: 'cdmhasocr_is', origin_path: 'cdmhasocr', formatters: [ToIFormatter]},
|
151
|
+
{dest_path: 'cdmisnewspaper_is', origin_path: 'cdmisnewspaper', formatters: [ToIFormatter]},
|
152
|
+
{dest_path: 'image_uri_ssi', origin_path: 'image_uri', formatters: [StripFormatter]},
|
153
|
+
{dest_path: 'record_type_ssi', origin_path: 'record_type', formatters: [StripFormatter]},
|
154
|
+
{dest_path: 'geographic_feature_ssim', origin_path: 'geogra', formatters: [Titlieze, StripFormatter, SplitFormatter, StripFormatter]},
|
155
|
+
{dest_path: 'geographic_feature_teim', origin_path: 'geogra', formatters: [StripFormatter]},
|
156
|
+
{dest_path: 'geographic_feature_unstem_search', origin_path: 'geogra', formatters: [StripFormatter]},
|
157
|
+
{dest_path: 'geonam_ssi', origin_path: 'geonam', formatters: [StripFormatter]},
|
158
|
+
{dest_path: 'kaltura_audio_ssi', origin_path: 'audio', formatters: [StripFormatter]},
|
159
|
+
{dest_path: 'kaltura_audio_playlist_ssi', origin_path: 'audioa', formatters: [StripFormatter]},
|
160
|
+
{dest_path: 'kaltura_video_ssi', origin_path: 'video', formatters: [StripFormatter]},
|
161
|
+
{dest_path: 'kaltura_video_playlist_ssi', origin_path: 'videoa', formatters: [StripFormatter]},
|
162
|
+
{dest_path: 'coordinates_llsi', origin_path: 'geonam', formatters: [GeoNameID, GeoNameIDToJson, GeoNameToLocation]},
|
163
|
+
{dest_path: 'placename_ssim', origin_path: 'geonam', formatters: [GeoNameID, GeoNameIDToJson, GeoNameToPlaceName]},
|
164
|
+
{dest_path: 'placename_unstem_search', origin_path: 'geonam', formatters: [GeoNameID, GeoNameIDToJson, GeoNameToPlaceName]},
|
165
|
+
{dest_path: 'table_ssim', origin_path: 'table', formatters: [StripFormatter, SplitFormatter, StripFormatter]},
|
166
|
+
{dest_path: 'umedia_ssi', origin_path: 'umedia', formatters: [StripFormatter]},
|
167
|
+
{dest_path: 'child_index', origin_path: 'child_index', formatters: []}
|
168
|
+
]
|
169
|
+
end
|
170
|
+
end
|
171
|
+
end
|
data/lib/cdmdexer.rb
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'contentdm_api'
|
2
|
+
require 'forwardable'
|
3
|
+
|
4
|
+
require 'cdmdexer/version'
|
5
|
+
require 'cdmdexer/oai_request'
|
6
|
+
require 'cdmdexer/formatters'
|
7
|
+
require 'cdmdexer/field_transformer'
|
8
|
+
require 'cdmdexer/field_formatter'
|
9
|
+
require 'cdmdexer/record_transformer'
|
10
|
+
require 'cdmdexer/transformer'
|
11
|
+
require 'cdmdexer/default_solr'
|
12
|
+
require 'cdmdexer/loader'
|
13
|
+
require 'cdmdexer/etl_worker'
|
14
|
+
require 'cdmdexer/default_completed_callback'
|
15
|
+
require 'cdmdexer/default_cdm_notification'
|
16
|
+
require 'cdmdexer/default_oai_notification'
|
17
|
+
require 'cdmdexer/default_loader_notification'
|
18
|
+
require 'cdmdexer/hooks'
|
19
|
+
require 'cdmdexer/oai_client'
|
20
|
+
require 'cdmdexer/load_worker'
|
21
|
+
require 'cdmdexer/transform_worker'
|
22
|
+
require 'cdmdexer/filtered_set_specs'
|
23
|
+
require 'cdmdexer/etl_by_set_specs'
|
24
|
+
require 'cdmdexer/regex_filter_callback'
|
25
|
+
require 'cdmdexer/field_mapping'
|
26
|
+
require 'cdmdexer/cdm_item'
|
data/travis.yml
ADDED
metadata
ADDED
@@ -0,0 +1,223 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: cdmdexer
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.17.7
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- chadfennell
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2019-03-13 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: hash_at_path
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0.1'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0.1'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: contentdm_api
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 0.5.0
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 0.5.0
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: sidekiq
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '3.5'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '3.5'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: titleize
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '1.4'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '1.4'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: rsolr
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '2.0'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '2.0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: rails
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '5.2'
|
90
|
+
type: :runtime
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '5.2'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: bundler
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '1.12'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - "~>"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '1.12'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: rake
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - "~>"
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '12.0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - "~>"
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '12.0'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: minitest
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - "~>"
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '5.0'
|
132
|
+
type: :development
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - "~>"
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: '5.0'
|
139
|
+
- !ruby/object:Gem::Dependency
|
140
|
+
name: yard
|
141
|
+
requirement: !ruby/object:Gem::Requirement
|
142
|
+
requirements:
|
143
|
+
- - "~>"
|
144
|
+
- !ruby/object:Gem::Version
|
145
|
+
version: 0.9.0
|
146
|
+
type: :development
|
147
|
+
prerelease: false
|
148
|
+
version_requirements: !ruby/object:Gem::Requirement
|
149
|
+
requirements:
|
150
|
+
- - "~>"
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
version: 0.9.0
|
153
|
+
description:
|
154
|
+
email:
|
155
|
+
- fenne035@umn.edu
|
156
|
+
executables: []
|
157
|
+
extensions: []
|
158
|
+
extra_rdoc_files: []
|
159
|
+
files:
|
160
|
+
- ".gitignore"
|
161
|
+
- ".rubocop.yml"
|
162
|
+
- ".travis.yml"
|
163
|
+
- CODE_OF_CONDUCT.md
|
164
|
+
- Gemfile
|
165
|
+
- LICENSE.txt
|
166
|
+
- README.md
|
167
|
+
- Rakefile
|
168
|
+
- bin/console
|
169
|
+
- bin/setup
|
170
|
+
- cdmdexer.gemspec
|
171
|
+
- lib/cdmdexer.rb
|
172
|
+
- lib/cdmdexer/cdm_item.rb
|
173
|
+
- lib/cdmdexer/default_cdm_notification.rb
|
174
|
+
- lib/cdmdexer/default_completed_callback.rb
|
175
|
+
- lib/cdmdexer/default_loader_notification.rb
|
176
|
+
- lib/cdmdexer/default_oai_notification.rb
|
177
|
+
- lib/cdmdexer/default_solr.rb
|
178
|
+
- lib/cdmdexer/etl_by_set_specs.rb
|
179
|
+
- lib/cdmdexer/etl_worker.rb
|
180
|
+
- lib/cdmdexer/field_formatter.rb
|
181
|
+
- lib/cdmdexer/field_mapping.rb
|
182
|
+
- lib/cdmdexer/field_transformer.rb
|
183
|
+
- lib/cdmdexer/filtered_set_specs.rb
|
184
|
+
- lib/cdmdexer/formatters.rb
|
185
|
+
- lib/cdmdexer/hooks.rb
|
186
|
+
- lib/cdmdexer/load_worker.rb
|
187
|
+
- lib/cdmdexer/loader.rb
|
188
|
+
- lib/cdmdexer/oai_client.rb
|
189
|
+
- lib/cdmdexer/oai_request.rb
|
190
|
+
- lib/cdmdexer/rake_task.rb
|
191
|
+
- lib/cdmdexer/record_transformer.rb
|
192
|
+
- lib/cdmdexer/regex_filter_callback.rb
|
193
|
+
- lib/cdmdexer/tasks/delete.rake
|
194
|
+
- lib/cdmdexer/tasks/etl.rake
|
195
|
+
- lib/cdmdexer/transform_worker.rb
|
196
|
+
- lib/cdmdexer/transformer.rb
|
197
|
+
- lib/cdmdexer/version.rb
|
198
|
+
- travis.yml
|
199
|
+
homepage:
|
200
|
+
licenses:
|
201
|
+
- MIT
|
202
|
+
metadata: {}
|
203
|
+
post_install_message:
|
204
|
+
rdoc_options: []
|
205
|
+
require_paths:
|
206
|
+
- lib
|
207
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
208
|
+
requirements:
|
209
|
+
- - ">="
|
210
|
+
- !ruby/object:Gem::Version
|
211
|
+
version: '0'
|
212
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
213
|
+
requirements:
|
214
|
+
- - ">="
|
215
|
+
- !ruby/object:Gem::Version
|
216
|
+
version: '0'
|
217
|
+
requirements: []
|
218
|
+
rubygems_version: 3.0.3
|
219
|
+
signing_key:
|
220
|
+
specification_version: 4
|
221
|
+
summary: Load CONTENTdm data into a Solr Index. CDMDEXER expects to run inside a Rails
|
222
|
+
application.
|
223
|
+
test_files: []
|