bulkrax 1.0.1 → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/controllers/bulkrax/importers_controller.rb +2 -2
- data/app/factories/bulkrax/object_factory.rb +10 -4
- data/app/matchers/bulkrax/application_matcher.rb +4 -1
- data/app/models/bulkrax/csv_entry.rb +37 -3
- data/app/models/bulkrax/importer.rb +10 -0
- data/app/models/concerns/bulkrax/file_factory.rb +2 -1
- data/app/models/concerns/bulkrax/has_matchers.rb +15 -3
- data/app/views/bulkrax/exporters/show.html.erb +11 -0
- data/app/views/bulkrax/importers/_edit_form_buttons.html.erb +45 -14
- data/app/views/bulkrax/importers/edit.html.erb +2 -0
- data/lib/bulkrax/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2cad330885f5f4d2dce25c989ebc7552b49b2212b68434d5def942fff61ad23a
|
4
|
+
data.tar.gz: 2b1e71044b26ff6ab9b25822ee969a7e886e095bbd2147b9b240406634e8f832
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 18e70353c64be13d686ff1da35d01c932422da49f9fd0ebf08470fdc0edc9333a9b51633b158cfaedf020ae5e3830574eb192a69ba498dff81e92847de4235f7
|
7
|
+
data.tar.gz: 6502ec16bc63aefed3c678ec062d3bc923c159260ac39278e692f4628bfe05acd675d49193b92c9e42f2ab50976d48c3d608d77f8db1d45774a664fd74b09bce
|
@@ -118,12 +118,12 @@ module Bulkrax
|
|
118
118
|
# OAI-only - selective re-harvest
|
119
119
|
elsif params[:commit] == 'Update and Harvest Updated Items'
|
120
120
|
Bulkrax::ImporterJob.perform_later(@importer.id, true)
|
121
|
-
elsif params[:commit] == 'Update
|
121
|
+
elsif params[:commit] == 'Update Metadata and Files'
|
122
122
|
@importer.parser_fields['update_files'] = true
|
123
123
|
@importer.save
|
124
124
|
Bulkrax::ImporterJob.perform_later(@importer.id)
|
125
125
|
# Perform a full metadata and files re-import; do the same for an OAI re-harvest of all items
|
126
|
-
elsif params[:commit] == ('Update and Replace
|
126
|
+
elsif params[:commit] == ('Update and Replace Files' || 'Update and Re-Harvest All Items')
|
127
127
|
@importer.parser_fields['replace_files'] = true
|
128
128
|
@importer.save
|
129
129
|
Bulkrax::ImporterJob.perform_later(@importer.id)
|
@@ -19,6 +19,11 @@ module Bulkrax
|
|
19
19
|
end
|
20
20
|
# rubocop:enable Metrics/ParameterLists
|
21
21
|
|
22
|
+
# update files is set, replace files is set or this is a create
|
23
|
+
def with_files
|
24
|
+
update_files || replace_files || !object
|
25
|
+
end
|
26
|
+
|
22
27
|
def run
|
23
28
|
arg_hash = { id: attributes[:id], name: 'UPDATE', klass: klass }
|
24
29
|
@object = find
|
@@ -204,10 +209,10 @@ module Bulkrax
|
|
204
209
|
def attribute_update
|
205
210
|
return transform_attributes.except(:id) if klass == Collection
|
206
211
|
if attributes[:collection].present?
|
207
|
-
transform_attributes.except(:id).except(:collection).merge(member_of_collections_attributes: { 0 => { id: collection.id } })
|
212
|
+
transform_attributes.except(:id).except(:collection).merge(member_of_collections_attributes: { "0" => { id: collection.id } })
|
208
213
|
elsif attributes[:collections].present?
|
209
214
|
collection_ids = attributes[:collections].each.with_index.each_with_object({}) do |(element, index), ids|
|
210
|
-
ids[index] = element
|
215
|
+
ids[index.to_s] = element
|
211
216
|
end
|
212
217
|
transform_attributes.except(:id).except(:collections).merge(member_of_collections_attributes: collection_ids)
|
213
218
|
else
|
@@ -218,8 +223,9 @@ module Bulkrax
|
|
218
223
|
# Override if we need to map the attributes from the parser in
|
219
224
|
# a way that is compatible with how the factory needs them.
|
220
225
|
def transform_attributes
|
221
|
-
attributes.slice(*permitted_attributes)
|
222
|
-
|
226
|
+
@transform_attributes = attributes.slice(*permitted_attributes)
|
227
|
+
@transform_attributes.merge!(file_attributes(update_files)) if with_files
|
228
|
+
@transform_attributes
|
223
229
|
end
|
224
230
|
|
225
231
|
# Regardless of what the Parser gives us, these are the properties we are prepared to accept.
|
@@ -97,9 +97,17 @@ module Bulkrax
|
|
97
97
|
mapping.each do |key, value|
|
98
98
|
next if Bulkrax.reserved_properties.include?(key) && !field_supported?(key)
|
99
99
|
next if key == "model"
|
100
|
-
|
101
|
-
|
102
|
-
|
100
|
+
|
101
|
+
object_key = key if value.key?('object')
|
102
|
+
next unless hyrax_record.respond_to?(key.to_s) || object_key.present?
|
103
|
+
|
104
|
+
data = object_key.present? ? hyrax_record.send(value['object']) : hyrax_record.send(key.to_s)
|
105
|
+
if object_key.present?
|
106
|
+
next self.parsed_metadata[key] = '' if data.empty?
|
107
|
+
data = data.first if data.is_a?(ActiveTriples::Relation)
|
108
|
+
|
109
|
+
object_metadata(data, object_key)
|
110
|
+
elsif data.is_a?(ActiveTriples::Relation)
|
103
111
|
self.parsed_metadata[key] = data.map { |d| prepare_export_data(d) }.join('; ').to_s unless value[:excluded]
|
104
112
|
else
|
105
113
|
self.parsed_metadata[key] = prepare_export_data(data)
|
@@ -115,6 +123,32 @@ module Bulkrax
|
|
115
123
|
end
|
116
124
|
end
|
117
125
|
|
126
|
+
def object_metadata(data, object_key)
|
127
|
+
data = convert_to_hash(data)
|
128
|
+
|
129
|
+
data.each_with_index do |obj, index|
|
130
|
+
next unless obj[object_key]
|
131
|
+
|
132
|
+
next self.parsed_metadata["#{object_key}_#{index + 1}"] = prepare_export_data(obj[object_key]) unless obj[object_key].is_a?(Array)
|
133
|
+
|
134
|
+
obj[object_key].each_with_index do |_nested_item, nested_index|
|
135
|
+
self.parsed_metadata["#{object_key}_#{index + 1}_#{nested_index + 1}"] = prepare_export_data(obj[object_key][nested_index])
|
136
|
+
end
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
def convert_to_hash(data)
|
141
|
+
# converts data from `'[{}]'` to `[{}]`
|
142
|
+
gsub_data = data.gsub(/\[{/, '{')
|
143
|
+
.gsub(/}\]/, '}')
|
144
|
+
.gsub('=>', ':')
|
145
|
+
.gsub(/},\s?{/, "}},{{")
|
146
|
+
.split("},{")
|
147
|
+
gsub_data = [gsub_data] if gsub_data.is_a?(String)
|
148
|
+
|
149
|
+
return gsub_data.map { |d| JSON.parse(d) }
|
150
|
+
end
|
151
|
+
|
118
152
|
# In order for the existing exported hyrax_record, to be updated by a re-import
|
119
153
|
# we need a unique value in system_identifier
|
120
154
|
# add the existing hyrax_record id to system_identifier
|
@@ -24,6 +24,16 @@ module Bulkrax
|
|
24
24
|
attr_accessor :only_updates, :file_style, :file
|
25
25
|
attr_writer :current_run
|
26
26
|
|
27
|
+
def self.safe_uri_filename(uri)
|
28
|
+
uri = URI.parse(uri) unless uri.is_a?(URI)
|
29
|
+
r = Faraday.head(uri.to_s)
|
30
|
+
return CGI.parse(r.headers['content-disposition'])["filename"][0].delete("\"")
|
31
|
+
rescue
|
32
|
+
filename = File.basename(uri.path)
|
33
|
+
filename.delete!('/')
|
34
|
+
filename.presence || file_set.id
|
35
|
+
end
|
36
|
+
|
27
37
|
def status
|
28
38
|
if self.validate_only
|
29
39
|
'Validated'
|
@@ -33,7 +33,8 @@ module Bulkrax
|
|
33
33
|
if file_value.is_a?(Hash)
|
34
34
|
file_value
|
35
35
|
elsif file_value.is_a?(String)
|
36
|
-
|
36
|
+
name = Bulkrax::Importer.safe_uri_filename(file_value)
|
37
|
+
{ url: file_value, file_name: name }
|
37
38
|
else
|
38
39
|
Rails.logger.error("skipped remote file #{file_value} because we do not recognize the type")
|
39
40
|
nil
|
@@ -1,5 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
2
|
+
# rubocop:disable Metrics/ModuleLength
|
3
3
|
module Bulkrax
|
4
4
|
module HasMatchers
|
5
5
|
extend ActiveSupport::Concern
|
@@ -90,9 +90,20 @@ module Bulkrax
|
|
90
90
|
end
|
91
91
|
|
92
92
|
def multiple_metadata(content)
|
93
|
-
content = content.content if content.is_a?(Nokogiri::XML::NodeSet)
|
94
93
|
return unless content
|
95
|
-
|
94
|
+
|
95
|
+
case content
|
96
|
+
when Nokogiri::XML::NodeSet
|
97
|
+
content&.content
|
98
|
+
when Array
|
99
|
+
content
|
100
|
+
when Hash
|
101
|
+
Array.wrap(content)
|
102
|
+
when String
|
103
|
+
Array.wrap(content.strip)
|
104
|
+
else
|
105
|
+
Array.wrap(content)
|
106
|
+
end
|
96
107
|
end
|
97
108
|
|
98
109
|
def matched_metadata(multiple, name, result, object_multiple)
|
@@ -153,3 +164,4 @@ module Bulkrax
|
|
153
164
|
end
|
154
165
|
end
|
155
166
|
end
|
167
|
+
# rubocop:enable Metrics/ModuleLength
|
@@ -7,6 +7,13 @@
|
|
7
7
|
<div class='panel panel-default'>
|
8
8
|
<div class='panel-body'>
|
9
9
|
|
10
|
+
<% if File.exist?(@exporter.exporter_export_zip_path) %>
|
11
|
+
<p class='bulkrax-p-align'>
|
12
|
+
<strong>Download:</strong>
|
13
|
+
<%= link_to raw('<span class="glyphicon glyphicon-download"></span>'), exporter_download_path(@exporter) %>
|
14
|
+
</p>
|
15
|
+
<% end %>
|
16
|
+
|
10
17
|
<p class='bulkrax-p-align'>
|
11
18
|
<strong><%= t('bulkrax.exporter.labels.name') %>:</strong>
|
12
19
|
<%= @exporter.name %>
|
@@ -116,6 +123,10 @@
|
|
116
123
|
<%= page_entries_info(@work_entries) %><br>
|
117
124
|
<%= paginate(@work_entries, param_name: :work_entries_page) %>
|
118
125
|
<br>
|
126
|
+
<% if File.exist?(@exporter.exporter_export_zip_path) %>
|
127
|
+
<%= link_to 'Download', exporter_download_path(@exporter) %>
|
128
|
+
|
|
129
|
+
<% end %>
|
119
130
|
<%= link_to 'Edit', edit_exporter_path(@exporter) %>
|
120
131
|
|
|
121
132
|
<%= link_to 'Back', exporters_path %>
|
@@ -1,16 +1,47 @@
|
|
1
|
+
<div class="modal fade" id="bulkraxModal" tabindex="-1" role="dialog" aria-labelledby="bulkraxModalLabel">
|
2
|
+
<div class="modal-dialog" role="document">
|
3
|
+
<div class="modal-content">
|
4
|
+
<div class="modal-body">
|
5
|
+
<h5>Options for Updating the Importer</h5>
|
6
|
+
<hr />
|
1
7
|
|
2
|
-
|
8
|
+
<% if @importer.importer_runs.blank? %>
|
9
|
+
<p>Only update the values in the importer form. Do not import metadata or files for any works or collections.</p>
|
10
|
+
<%= form.button :submit, value: 'Update Importer', class: 'btn btn-primary' %>
|
11
|
+
<hr />
|
12
|
+
<p>Update the values in the importer form and run the importer for the first time.</p>
|
13
|
+
<%= form.button :submit, value: 'Update and Import', class: 'btn btn-primary' %>
|
14
|
+
<% elsif @importer.parser_klass.include?('Oai') %>
|
15
|
+
<p>Only update the values in the importer form. Do not update metadata or files for any works or collections.</p>
|
16
|
+
<%= form.button :submit, value: 'Update Importer', class: 'btn btn-primary' %>
|
17
|
+
<hr />
|
18
|
+
<p>Update the values in the importer form and update items that have changed at the source.</p>
|
19
|
+
<%= form.button :submit, value: 'Update and Harvest Updated Items', class: 'btn btn-primary' %>
|
20
|
+
<hr />
|
21
|
+
<p>Update the values in the importer form and recreate all items from the source.</p>
|
22
|
+
<%= form.button :submit, value: 'Update and Re-Harvest All Items', class: 'btn btn-primary' %>
|
23
|
+
<% else %>
|
24
|
+
<p>Only update the values in the importer form. Do not update metadata or files for any works or collections.</p>
|
25
|
+
<%= form.button :submit, value: 'Update Importer', class: 'btn btn-primary' %>
|
26
|
+
<hr />
|
27
|
+
<p>Update the values in the importer form and update the metadata for all works. Do not update any files.</p>
|
28
|
+
<%= form.button :submit, value: 'Update Metadata', class: 'btn btn-primary' %>
|
29
|
+
<hr />
|
30
|
+
<p>Update the values in the importer form and update the metadata and files for all works. Creates new versions of the files and retains the old versions.</p>
|
31
|
+
<%= form.button :submit, value: 'Update Metadata and Files', class: 'btn btn-primary' %>
|
32
|
+
<hr />
|
33
|
+
<p>Update the values in the importer form and update the metadata. Completely removes all files attached to works for this importer and recreates the files from scratch.</p>
|
34
|
+
<%= form.button :submit,
|
35
|
+
value: 'Update and Replace Files',
|
36
|
+
class: 'btn btn-primary',
|
37
|
+
data: {confirm: "Are you sure? This will remove all files before adding them from the import."} %>
|
38
|
+
<% end %>
|
39
|
+
<hr />
|
3
40
|
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
<% else %>
|
12
|
-
<%= form.button :submit, value: 'Update Importer', class: 'btn btn-primary' %>
|
13
|
-
| <%= form.button :submit, value: 'Update and Re-Import (update metadata only)', class: 'btn btn-primary' %>
|
14
|
-
| <%= form.button :submit, value: 'Update All (update metadata and update files)', class: 'btn btn-primary' %>
|
15
|
-
| <%= form.button :submit, value: 'Update and Re-Import (update metadata and replace files)', class: 'btn btn-primary', data: {confirm: "Are you sure? This will remove all files before adding them from the import."} %>
|
16
|
-
<% end %>
|
41
|
+
</div>
|
42
|
+
<div class="modal-footer">
|
43
|
+
<button type="button" class="btn btn-default" data-dismiss="modal"><%= t('helpers.action.cancel') %></button>
|
44
|
+
</div>
|
45
|
+
</div>
|
46
|
+
</div>
|
47
|
+
</div>
|
@@ -5,10 +5,12 @@
|
|
5
5
|
<div class="row">
|
6
6
|
<div class="col-md-12">
|
7
7
|
<div class="panel panel-default tabs">
|
8
|
+
|
8
9
|
<%= simple_form_for @importer, html: { multipart: true } do |form| %>
|
9
10
|
<%= render 'form', importer: @importer, form: form %>
|
10
11
|
<div class="panel-footer">
|
11
12
|
<div class='pull-right'>
|
13
|
+
<%= link_to 'Update Importer', '#bulkraxModal', class: "btn btn-primary", data: { toggle: 'modal' } %>
|
12
14
|
<%= render 'edit_form_buttons', form: form %>
|
13
15
|
<% cancel_path = form.object.persisted? ? importer_path(form.object) : importers_path %>
|
14
16
|
| <%= link_to t('.cancel'), cancel_path, class: 'btn btn-default ' %>
|
data/lib/bulkrax/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: bulkrax
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Rob Kaufman
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-08-
|
11
|
+
date: 2021-08-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rails
|