bulkrax 1.0.0 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/app/controllers/bulkrax/exporters_controller.rb +12 -4
- data/app/controllers/bulkrax/importers_controller.rb +22 -17
- data/app/factories/bulkrax/object_factory.rb +44 -61
- data/app/jobs/bulkrax/create_relationships_job.rb +187 -0
- data/app/jobs/bulkrax/delete_work_job.rb +6 -2
- data/app/jobs/bulkrax/export_work_job.rb +3 -1
- data/app/jobs/bulkrax/exporter_job.rb +1 -0
- data/app/jobs/bulkrax/{import_work_collection_job.rb → import_collection_job.rb} +2 -2
- data/app/jobs/bulkrax/importer_job.rb +16 -1
- data/app/matchers/bulkrax/application_matcher.rb +9 -6
- data/app/models/bulkrax/csv_collection_entry.rb +8 -6
- data/app/models/bulkrax/csv_entry.rb +139 -45
- data/app/models/bulkrax/entry.rb +19 -8
- data/app/models/bulkrax/exporter.rb +12 -5
- data/app/models/bulkrax/importer.rb +22 -5
- data/app/models/bulkrax/oai_entry.rb +5 -1
- data/app/models/bulkrax/rdf_entry.rb +16 -7
- data/app/models/bulkrax/xml_entry.rb +4 -0
- data/app/models/concerns/bulkrax/export_behavior.rb +2 -2
- data/app/models/concerns/bulkrax/file_factory.rb +2 -1
- data/app/models/concerns/bulkrax/has_matchers.rb +59 -16
- data/app/models/concerns/bulkrax/import_behavior.rb +35 -5
- data/app/models/concerns/bulkrax/importer_exporter_behavior.rb +19 -0
- data/app/models/concerns/bulkrax/status_info.rb +4 -4
- data/app/parsers/bulkrax/application_parser.rb +59 -84
- data/app/parsers/bulkrax/bagit_parser.rb +12 -3
- data/app/parsers/bulkrax/csv_parser.rb +117 -62
- data/app/parsers/bulkrax/oai_dc_parser.rb +5 -2
- data/app/parsers/bulkrax/xml_parser.rb +5 -0
- data/app/views/bulkrax/exporters/_form.html.erb +1 -1
- data/app/views/bulkrax/exporters/show.html.erb +13 -1
- data/app/views/bulkrax/importers/_edit_form_buttons.html.erb +45 -14
- data/app/views/bulkrax/importers/edit.html.erb +2 -0
- data/app/views/bulkrax/importers/index.html.erb +15 -17
- data/app/views/bulkrax/importers/show.html.erb +6 -2
- data/config/locales/bulkrax.en.yml +1 -0
- data/db/migrate/20190731114016_change_importer_and_exporter_to_polymorphic.rb +5 -1
- data/db/migrate/20211004170708_change_bulkrax_statuses_error_message_column_type_to_text.rb +5 -0
- data/db/migrate/20211203195233_rename_children_counters_to_relationships.rb +6 -0
- data/lib/bulkrax/engine.rb +1 -1
- data/lib/bulkrax/version.rb +1 -1
- data/lib/bulkrax.rb +9 -17
- data/lib/generators/bulkrax/templates/bin/importer +17 -11
- data/lib/generators/bulkrax/templates/config/bulkrax_api.yml +3 -1
- data/lib/generators/bulkrax/templates/config/initializers/bulkrax.rb +7 -12
- metadata +13 -7
- data/app/jobs/bulkrax/child_relationships_job.rb +0 -128
@@ -12,6 +12,11 @@ module Bulkrax
|
|
12
12
|
# @todo not yet supported
|
13
13
|
def create_collections; end
|
14
14
|
|
15
|
+
# TODO: change to differentiate between collection and work records when adding ability to import collection metadata
|
16
|
+
def works_total
|
17
|
+
total
|
18
|
+
end
|
19
|
+
|
15
20
|
# @todo not yet supported
|
16
21
|
def import_fields; end
|
17
22
|
|
@@ -32,7 +32,7 @@
|
|
32
32
|
prompt: 'Select from the list',
|
33
33
|
label_html: { class: 'importer export-source-option hidden' },
|
34
34
|
input_html: { class: 'importer export-source-option hidden' },
|
35
|
-
collection: form.object.importers_list %>
|
35
|
+
collection: form.object.importers_list.sort %>
|
36
36
|
|
37
37
|
<%= form.input :export_source_collection,
|
38
38
|
prompt: 'Start typing ...',
|
@@ -7,6 +7,13 @@
|
|
7
7
|
<div class='panel panel-default'>
|
8
8
|
<div class='panel-body'>
|
9
9
|
|
10
|
+
<% if File.exist?(@exporter.exporter_export_zip_path) %>
|
11
|
+
<p class='bulkrax-p-align'>
|
12
|
+
<strong>Download:</strong>
|
13
|
+
<%= link_to raw('<span class="glyphicon glyphicon-download"></span>'), exporter_download_path(@exporter) %>
|
14
|
+
</p>
|
15
|
+
<% end %>
|
16
|
+
|
10
17
|
<p class='bulkrax-p-align'>
|
11
18
|
<strong><%= t('bulkrax.exporter.labels.name') %>:</strong>
|
12
19
|
<%= @exporter.name %>
|
@@ -50,8 +57,9 @@
|
|
50
57
|
<strong><%= t('bulkrax.exporter.labels.limit') %>:</strong>
|
51
58
|
<%= @exporter.limit %>
|
52
59
|
</p>
|
60
|
+
<%= render partial: 'bulkrax/shared/bulkrax_errors', locals: {item: @exporter} %>
|
53
61
|
|
54
|
-
|
62
|
+
<%= render partial: 'bulkrax/shared/bulkrax_field_mapping', locals: {item: @exporter} %>
|
55
63
|
|
56
64
|
<%# Currently, no parser-specific fields exist on Exporter,
|
57
65
|
thus there's no real reason to always show this field %>
|
@@ -116,6 +124,10 @@
|
|
116
124
|
<%= page_entries_info(@work_entries) %><br>
|
117
125
|
<%= paginate(@work_entries, param_name: :work_entries_page) %>
|
118
126
|
<br>
|
127
|
+
<% if File.exist?(@exporter.exporter_export_zip_path) %>
|
128
|
+
<%= link_to 'Download', exporter_download_path(@exporter) %>
|
129
|
+
|
|
130
|
+
<% end %>
|
119
131
|
<%= link_to 'Edit', edit_exporter_path(@exporter) %>
|
120
132
|
|
|
121
133
|
<%= link_to 'Back', exporters_path %>
|
@@ -1,16 +1,47 @@
|
|
1
|
+
<div class="modal fade" id="bulkraxModal" tabindex="-1" role="dialog" aria-labelledby="bulkraxModalLabel">
|
2
|
+
<div class="modal-dialog" role="document">
|
3
|
+
<div class="modal-content">
|
4
|
+
<div class="modal-body">
|
5
|
+
<h5>Options for Updating the Importer</h5>
|
6
|
+
<hr />
|
1
7
|
|
2
|
-
|
8
|
+
<% if @importer.importer_runs.blank? %>
|
9
|
+
<p>Only update the values in the importer form. Do not import metadata or files for any works or collections.</p>
|
10
|
+
<%= form.button :submit, value: 'Update Importer', class: 'btn btn-primary' %>
|
11
|
+
<hr />
|
12
|
+
<p>Update the values in the importer form and run the importer for the first time.</p>
|
13
|
+
<%= form.button :submit, value: 'Update and Import', class: 'btn btn-primary' %>
|
14
|
+
<% elsif @importer.parser_klass.include?('Oai') %>
|
15
|
+
<p>Only update the values in the importer form. Do not update metadata or files for any works or collections.</p>
|
16
|
+
<%= form.button :submit, value: 'Update Importer', class: 'btn btn-primary' %>
|
17
|
+
<hr />
|
18
|
+
<p>Update the values in the importer form and update items that have changed at the source.</p>
|
19
|
+
<%= form.button :submit, value: 'Update and Harvest Updated Items', class: 'btn btn-primary' %>
|
20
|
+
<hr />
|
21
|
+
<p>Update the values in the importer form and recreate all items from the source.</p>
|
22
|
+
<%= form.button :submit, value: 'Update and Re-Harvest All Items', class: 'btn btn-primary' %>
|
23
|
+
<% else %>
|
24
|
+
<p>Only update the values in the importer form. Do not update metadata or files for any works or collections.</p>
|
25
|
+
<%= form.button :submit, value: 'Update Importer', class: 'btn btn-primary' %>
|
26
|
+
<hr />
|
27
|
+
<p>Update the values in the importer form and update the metadata for all works. Do not update any files.</p>
|
28
|
+
<%= form.button :submit, value: 'Update Metadata', class: 'btn btn-primary' %>
|
29
|
+
<hr />
|
30
|
+
<p>Update the values in the importer form and update the metadata and files for all works. Creates new versions of the files and retains the old versions.</p>
|
31
|
+
<%= form.button :submit, value: 'Update Metadata and Files', class: 'btn btn-primary' %>
|
32
|
+
<hr />
|
33
|
+
<p>Update the values in the importer form and update the metadata. Completely removes all files attached to works for this importer and recreates the files from scratch.</p>
|
34
|
+
<%= form.button :submit,
|
35
|
+
value: 'Update and Replace Files',
|
36
|
+
class: 'btn btn-primary',
|
37
|
+
data: {confirm: "Are you sure? This will remove all files before adding them from the import."} %>
|
38
|
+
<% end %>
|
39
|
+
<hr />
|
3
40
|
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
<% else %>
|
12
|
-
<%= form.button :submit, value: 'Update Importer', class: 'btn btn-primary' %>
|
13
|
-
| <%= form.button :submit, value: 'Update and Re-Import (update metadata only)', class: 'btn btn-primary' %>
|
14
|
-
| <%= form.button :submit, value: 'Update All (update metadata and update files)', class: 'btn btn-primary' %>
|
15
|
-
| <%= form.button :submit, value: 'Update and Re-Import (update metadata and replace files)', class: 'btn btn-primary', data: {confirm: "Are you sure? This will remove all files before adding them from the import."} %>
|
16
|
-
<% end %>
|
41
|
+
</div>
|
42
|
+
<div class="modal-footer">
|
43
|
+
<button type="button" class="btn btn-default" data-dismiss="modal"><%= t('helpers.action.cancel') %></button>
|
44
|
+
</div>
|
45
|
+
</div>
|
46
|
+
</div>
|
47
|
+
</div>
|
@@ -5,10 +5,12 @@
|
|
5
5
|
<div class="row">
|
6
6
|
<div class="col-md-12">
|
7
7
|
<div class="panel panel-default tabs">
|
8
|
+
|
8
9
|
<%= simple_form_for @importer, html: { multipart: true } do |form| %>
|
9
10
|
<%= render 'form', importer: @importer, form: form %>
|
10
11
|
<div class="panel-footer">
|
11
12
|
<div class='pull-right'>
|
13
|
+
<%= link_to 'Update Importer', '#bulkraxModal', class: "btn btn-primary", data: { toggle: 'modal' } %>
|
12
14
|
<%= render 'edit_form_buttons', form: form %>
|
13
15
|
<% cancel_path = form.object.persisted? ? importer_path(form.object) : importers_path %>
|
14
16
|
| <%= link_to t('.cancel'), cancel_path, class: 'btn btn-default ' %>
|
@@ -31,23 +31,21 @@
|
|
31
31
|
</thead>
|
32
32
|
<tbody>
|
33
33
|
<% @importers.each do |importer| %>
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
<td><%= link_to raw('<span class="glyphicon glyphicon-remove"></span>'), importer, method: :delete, data: { confirm: 'Are you sure?' } %></td>
|
50
|
-
</tr>
|
34
|
+
<tr>
|
35
|
+
<th scope="row"><%= link_to importer.name, importer_path(importer) %></th>
|
36
|
+
<td><%= importer.status %></td>
|
37
|
+
<td><%= importer.last_imported_at.strftime("%b %d, %Y") if importer.last_imported_at %></td>
|
38
|
+
<td><%= importer.next_import_at.strftime("%b %d, %Y") if importer.next_import_at %></td>
|
39
|
+
<td><%= importer.importer_runs.last&.enqueued_records %></td>
|
40
|
+
<td><%= (importer.importer_runs.last&.processed_collections || 0) + (importer.importer_runs.last&.processed_records || 0) %></td>
|
41
|
+
<td><%= (importer.importer_runs.last&.failed_collections || 0) + (importer.importer_runs.last&.failed_records || 0) %></td>
|
42
|
+
<td><%= importer.importer_runs.last&.deleted_records %></td>
|
43
|
+
<td><%= importer.importer_runs.last&.total_collection_entries %></td>
|
44
|
+
<td><%= importer.importer_runs.last&.total_work_entries %></td>
|
45
|
+
<td><%= link_to raw('<span class="glyphicon glyphicon-info-sign"></span>'), importer_path(importer) %></td>
|
46
|
+
<td><%= link_to raw('<span class="glyphicon glyphicon-pencil"></span>'), edit_importer_path(importer) %></td>
|
47
|
+
<td><%= link_to raw('<span class="glyphicon glyphicon-remove"></span>'), importer, method: :delete, data: { confirm: 'Are you sure?' } %></td>
|
48
|
+
</tr>
|
51
49
|
<% end %>
|
52
50
|
</tbody>
|
53
51
|
</table>
|
@@ -101,8 +101,10 @@
|
|
101
101
|
<td><%= e.id %></td>
|
102
102
|
<% if e.status == "Complete" %>
|
103
103
|
<td><span class="glyphicon glyphicon-ok" style="color: green;"></span> <%= e.status %></td>
|
104
|
+
<% elsif e.status == "Pending" %>
|
105
|
+
<td><span class="glyphicon glyphicon-option-horizontal" style="color: blue;"></span> <%= e.status %></td>
|
104
106
|
<% else %>
|
105
|
-
<td><span class="glyphicon glyphicon-remove" style="color: red
|
107
|
+
<td><span class="glyphicon glyphicon-remove" style="color: <%= e.status == 'Deleted' ? 'green' : 'red' %>;"></span> <%= e.status %></td>
|
106
108
|
<% end %>
|
107
109
|
<% if e.last_error.present? %>
|
108
110
|
<td><%= link_to e.last_error.dig("error_class"), bulkrax.importer_entry_path(@importer.id, e.id) %></td>
|
@@ -137,8 +139,10 @@
|
|
137
139
|
<td><%= e.id %></td>
|
138
140
|
<% if e.status == "Complete" %>
|
139
141
|
<td><span class="glyphicon glyphicon-ok" style="color: green;"></span> <%= e.status %></td>
|
142
|
+
<% elsif e.status == "Pending" %>
|
143
|
+
<td><span class="glyphicon glyphicon-option-horizontal" style="color: blue;"></span> <%= e.status %></td>
|
140
144
|
<% else %>
|
141
|
-
<td><span class="glyphicon glyphicon-remove" style="color: red
|
145
|
+
<td><span class="glyphicon glyphicon-remove" style="color: <%= e.status == 'Deleted' ? 'green' : 'red' %>;"></span> <%= e.status %></td>
|
142
146
|
<% end %>
|
143
147
|
<% if e.last_error.present? %>
|
144
148
|
<td><%= link_to e.last_error.dig("error_class"), bulkrax.importer_entry_path(@importer.id, e.id) %></td>
|
@@ -6,7 +6,11 @@ end
|
|
6
6
|
|
7
7
|
class ChangeImporterAndExporterToPolymorphic < ActiveRecord::Migration[5.1]
|
8
8
|
def change
|
9
|
-
|
9
|
+
if column_exists?(:bulkrax_entries, :importer_id)
|
10
|
+
remove_foreign_key :bulkrax_entries, column: :importer_id
|
11
|
+
remove_index :bulkrax_entries, :importer_id
|
12
|
+
rename_column :bulkrax_entries, :importer_id, :importerexporter_id
|
13
|
+
end
|
10
14
|
add_column :bulkrax_entries, :importerexporter_type, :string, after: :id, default: 'Bulkrax::Importer' unless column_exists?(:bulkrax_entries, :importerexporter_type)
|
11
15
|
end
|
12
16
|
end
|
data/lib/bulkrax/engine.rb
CHANGED
@@ -25,7 +25,7 @@ module Bulkrax
|
|
25
25
|
config.after_initialize do
|
26
26
|
my_engine_root = Bulkrax::Engine.root.to_s
|
27
27
|
paths = ActionController::Base.view_paths.collect(&:to_s)
|
28
|
-
hyrax_path = paths.detect { |path| path.match(
|
28
|
+
hyrax_path = paths.detect { |path| path.match(/\/hyrax-[\d\.]+.*/) }
|
29
29
|
paths = if hyrax_path
|
30
30
|
paths.insert(paths.index(hyrax_path), my_engine_root + '/app/views')
|
31
31
|
else
|
data/lib/bulkrax/version.rb
CHANGED
data/lib/bulkrax.rb
CHANGED
@@ -5,12 +5,14 @@ require 'active_support/all'
|
|
5
5
|
|
6
6
|
module Bulkrax
|
7
7
|
class << self
|
8
|
+
# TODO: remove collection_field_mapping when releasing v2
|
8
9
|
mattr_accessor :parsers,
|
9
10
|
:default_work_type,
|
10
11
|
:default_field_mapping,
|
11
12
|
:collection_field_mapping,
|
12
13
|
:fill_in_blank_source_identifiers,
|
13
|
-
:
|
14
|
+
:related_children_field_mapping,
|
15
|
+
:related_parents_field_mapping,
|
14
16
|
:reserved_properties,
|
15
17
|
:field_mappings,
|
16
18
|
:import_path,
|
@@ -33,26 +35,16 @@ module Bulkrax
|
|
33
35
|
self.removed_image_path = Bulkrax::Engine.root.join('spec', 'fixtures', 'removed.png').to_s
|
34
36
|
self.server_name = 'bulkrax@example.com'
|
35
37
|
|
36
|
-
#
|
37
|
-
#
|
38
|
-
|
39
|
-
# Field_mapping for establishing a parent-child relationship (FROM parent TO child)
|
40
|
-
# This can be a Collection to Work, or Work to Work relationship
|
41
|
-
# This value IS NOT used for OAI, so setting the OAI Entries here will have no effect
|
42
|
-
# The mapping is supplied per Entry, provide the full class name as a string, eg. 'Bulkrax::CsvEntry'
|
43
|
-
# Example:
|
44
|
-
# {
|
45
|
-
# 'Bulkrax::RdfEntry' => 'http://opaquenamespace.org/ns/contents',
|
46
|
-
# 'Bulkrax::CsvEntry' => 'children'
|
47
|
-
# }
|
48
|
-
# By default no parent-child relationships are added
|
49
|
-
self.parent_child_field_mapping = {}
|
50
|
-
|
38
|
+
# NOTE: Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.
|
39
|
+
# Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.
|
40
|
+
# TODO: remove collection_field_mapping when releasing v2
|
51
41
|
# Field_mapping for establishing a collection relationship (FROM work TO collection)
|
52
42
|
# This value IS NOT used for OAI, so setting the OAI Entries here will have no effect
|
53
43
|
# The mapping is supplied per Entry, provide the full class name as a string, eg. 'Bulkrax::CsvEntry'
|
54
44
|
# The default value for CSV is collection
|
55
|
-
self.collection_field_mapping = {
|
45
|
+
self.collection_field_mapping = {
|
46
|
+
'Bulkrax::CsvEntry' => 'collection'
|
47
|
+
}
|
56
48
|
|
57
49
|
# Hash of Generic field_mappings for use in the view
|
58
50
|
# There must be one field_mappings hash per view parial
|
@@ -6,18 +6,11 @@ require_relative '../config/environment'
|
|
6
6
|
require 'slop'
|
7
7
|
|
8
8
|
def main(opts = {})
|
9
|
-
|
10
|
-
puts 'Missing required parameters'
|
11
|
-
help
|
12
|
-
end
|
13
|
-
|
14
|
-
if opts[:auth_token].blank?
|
15
|
-
puts 'Missing Authentication Token --auth_token'
|
16
|
-
exit
|
17
|
-
end
|
9
|
+
check_required_params
|
18
10
|
|
19
11
|
update = opts[:importer_id].present?
|
20
|
-
|
12
|
+
port = opts[:port].presence
|
13
|
+
url = build_url(opts.delete(:importer_id), opts.delete(:url), port)
|
21
14
|
|
22
15
|
headers = { 'Content-Type' => 'application/json' }
|
23
16
|
headers['Authorization'] = "Token: #{opts.delete(:auth_token)}"
|
@@ -43,6 +36,18 @@ def main(opts = {})
|
|
43
36
|
puts "#{response.status} - #{response.body.truncate(200)}"
|
44
37
|
end
|
45
38
|
|
39
|
+
def check_required_params
|
40
|
+
if opts[:importer_id].blank? && invalid?(opts)
|
41
|
+
puts 'Missing required parameters'
|
42
|
+
help
|
43
|
+
end
|
44
|
+
|
45
|
+
if opts[:auth_token].blank? # rubocop:disable Style/GuardClause
|
46
|
+
puts 'Missing Authentication Token --auth_token'
|
47
|
+
exit
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
46
51
|
def invalid?(opts)
|
47
52
|
required_params.each do |p|
|
48
53
|
return true if opts[p.to_sym].blank?
|
@@ -74,11 +79,12 @@ def build_params(opts = {})
|
|
74
79
|
return params.compact
|
75
80
|
end
|
76
81
|
|
77
|
-
def build_url(importer_id, url)
|
82
|
+
def build_url(importer_id, url, port = nil)
|
78
83
|
if url.nil?
|
79
84
|
protocol = Rails.application.config.force_ssl ? 'https://' : 'http://'
|
80
85
|
host = Rails.application.config.action_mailer.default_url_options[:host]
|
81
86
|
url = "#{protocol}#{host}"
|
87
|
+
url = "#{url}:#{port}" if port
|
82
88
|
end
|
83
89
|
path = Bulkrax::Engine.routes.url_helpers.polymorphic_path(Bulkrax::Importer)
|
84
90
|
url = File.join(url, path)
|
@@ -81,4 +81,6 @@ bulkrax:
|
|
81
81
|
- "Update and Import (importer has not yet been run)"
|
82
82
|
auth_token:
|
83
83
|
definition: 'Authentication token. Required for JSON requests only.'
|
84
|
-
required: true
|
84
|
+
required: true
|
85
|
+
port:
|
86
|
+
definition: 'Port to use in http request. Defaults to 80. May need to set to 3000 when running in the development environment'
|
@@ -19,18 +19,8 @@ Bulkrax.setup do |config|
|
|
19
19
|
# Server name for oai request header
|
20
20
|
# config.server_name = 'my_server@name.com'
|
21
21
|
|
22
|
-
#
|
23
|
-
#
|
24
|
-
# This value IS NOT used for OAI, so setting the OAI Entries here will have no effect
|
25
|
-
# The mapping is supplied per Entry, provide the full class name as a string, eg. 'Bulkrax::CsvEntry'
|
26
|
-
# Example:
|
27
|
-
# {
|
28
|
-
# 'Bulkrax::RdfEntry' => 'http://opaquenamespace.org/ns/contents',
|
29
|
-
# 'Bulkrax::CsvEntry' => 'children'
|
30
|
-
# }
|
31
|
-
# By default no parent-child relationships are added
|
32
|
-
# config.parent_child_field_mapping = { }
|
33
|
-
|
22
|
+
# NOTE: Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.
|
23
|
+
# Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.
|
34
24
|
# Field_mapping for establishing a collection relationship (FROM work TO collection)
|
35
25
|
# This value IS NOT used for OAI, so setting the OAI parser here will have no effect
|
36
26
|
# The mapping is supplied per Entry, provide the full class name as a string, eg. 'Bulkrax::CsvEntry'
|
@@ -48,6 +38,11 @@ Bulkrax.setup do |config|
|
|
48
38
|
# e.g. to exclude date
|
49
39
|
# config.field_mappings["Bulkrax::OaiDcParser"]["date"] = { from: ["date"], excluded: true }
|
50
40
|
#
|
41
|
+
# e.g. to import parent-child relationships
|
42
|
+
# config.field_mappings['Bulkrax::CsvParser']['parents'] = { from: ['parents'], related_parents_field_mapping: true }
|
43
|
+
# config.field_mappings['Bulkrax::CsvParser']['children'] = { from: ['children'], related_children_field_mapping: true }
|
44
|
+
# (For more info on importing relationships, see Bulkrax Wiki: https://github.com/samvera-labs/bulkrax/wiki/Configuring-Bulkrax#parent-child-relationship-field-mappings)
|
45
|
+
#
|
51
46
|
# # e.g. to add the required source_identifier field
|
52
47
|
# # config.field_mappings["Bulkrax::CsvParser"]["source_id"] = { from: ["old_source_id"], source_identifier: true }
|
53
48
|
# If you want Bulkrax to fill in source_identifiers for you, see below
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: bulkrax
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.0.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Rob Kaufman
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-01-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rails
|
@@ -224,7 +224,11 @@ dependencies:
|
|
224
224
|
- - ">="
|
225
225
|
- !ruby/object:Gem::Version
|
226
226
|
version: '0'
|
227
|
-
description:
|
227
|
+
description: Bulkrax is a batteries included importer for Samvera applications. It
|
228
|
+
currently includes support for OAI-PMH (DC and Qualified DC) and CSV out of the
|
229
|
+
box. It is also designed to be extensible, allowing you to easily add new importers
|
230
|
+
in to your application or to include them with other gems. Bulkrax provides a full
|
231
|
+
admin interface including creating, editing, scheduling and reviewing imports.
|
228
232
|
email:
|
229
233
|
- rob@notch8.com
|
230
234
|
executables: []
|
@@ -255,12 +259,12 @@ files:
|
|
255
259
|
- app/helpers/bulkrax/importers_helper.rb
|
256
260
|
- app/helpers/bulkrax/validation_helper.rb
|
257
261
|
- app/jobs/bulkrax/application_job.rb
|
258
|
-
- app/jobs/bulkrax/
|
262
|
+
- app/jobs/bulkrax/create_relationships_job.rb
|
259
263
|
- app/jobs/bulkrax/delete_work_job.rb
|
260
264
|
- app/jobs/bulkrax/download_cloud_file_job.rb
|
261
265
|
- app/jobs/bulkrax/export_work_job.rb
|
262
266
|
- app/jobs/bulkrax/exporter_job.rb
|
263
|
-
- app/jobs/bulkrax/
|
267
|
+
- app/jobs/bulkrax/import_collection_job.rb
|
264
268
|
- app/jobs/bulkrax/import_work_job.rb
|
265
269
|
- app/jobs/bulkrax/importer_job.rb
|
266
270
|
- app/mailers/bulkrax/application_mailer.rb
|
@@ -351,6 +355,8 @@ files:
|
|
351
355
|
- db/migrate/20201117220007_add_workflow_status_to_bulkrax_exporter.rb
|
352
356
|
- db/migrate/20210806044408_remove_unused_last_error.rb
|
353
357
|
- db/migrate/20210806065737_increase_text_sizes.rb
|
358
|
+
- db/migrate/20211004170708_change_bulkrax_statuses_error_message_column_type_to_text.rb
|
359
|
+
- db/migrate/20211203195233_rename_children_counters_to_relationships.rb
|
354
360
|
- lib/bulkrax.rb
|
355
361
|
- lib/bulkrax/engine.rb
|
356
362
|
- lib/bulkrax/version.rb
|
@@ -381,8 +387,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
381
387
|
- !ruby/object:Gem::Version
|
382
388
|
version: '0'
|
383
389
|
requirements: []
|
384
|
-
rubygems_version: 3.
|
390
|
+
rubygems_version: 3.1.2
|
385
391
|
signing_key:
|
386
392
|
specification_version: 4
|
387
|
-
summary:
|
393
|
+
summary: Import and export tool for Hyrax and Hyku
|
388
394
|
test_files: []
|
@@ -1,128 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Bulkrax
|
4
|
-
class ChildWorksError < RuntimeError; end
|
5
|
-
class ChildRelationshipsJob < ApplicationJob
|
6
|
-
queue_as :import
|
7
|
-
|
8
|
-
def perform(*args)
|
9
|
-
@args = args
|
10
|
-
|
11
|
-
if entry.factory_class == Collection
|
12
|
-
collection_membership
|
13
|
-
else
|
14
|
-
work_membership
|
15
|
-
end
|
16
|
-
# Not all of the Works/Collections exist yet; reschedule
|
17
|
-
rescue Bulkrax::ChildWorksError
|
18
|
-
reschedule(args[0], args[1], args[2])
|
19
|
-
end
|
20
|
-
|
21
|
-
def collection_membership
|
22
|
-
# add collection to works
|
23
|
-
member_of_collection = []
|
24
|
-
child_works_hash.each { |k, v| member_of_collection << k if v[:class_name] != 'Collection' }
|
25
|
-
member_of_collection.each { |work| work_child_collection_parent(work) }
|
26
|
-
|
27
|
-
# add collections to collection
|
28
|
-
members_collections = []
|
29
|
-
child_works_hash.each { |k, v| members_collections << k if v[:class_name] == 'Collection' }
|
30
|
-
collection_parent_collection_child(members_collections) if members_collections.present?
|
31
|
-
end
|
32
|
-
|
33
|
-
def work_membership
|
34
|
-
# add works to work
|
35
|
-
# reject any Collections, they can't be children of Works
|
36
|
-
members_works = []
|
37
|
-
# reject any Collections, they can't be children of Works
|
38
|
-
child_works_hash.each { |k, v| members_works << k if v[:class_name] != 'Collection' }
|
39
|
-
if members_works.length < child_entries.length # rubocop:disable Style/IfUnlessModifier
|
40
|
-
Rails.logger.warn("Cannot add collections as children of works: #{(@child_entries.length - members_works.length)} collections were discarded for parent entry #{@entry.id} (of #{@child_entries.length})")
|
41
|
-
end
|
42
|
-
work_parent_work_child(members_works) if members_works.present?
|
43
|
-
end
|
44
|
-
|
45
|
-
def entry
|
46
|
-
@entry ||= Bulkrax::Entry.find(@args[0])
|
47
|
-
end
|
48
|
-
|
49
|
-
def child_entries
|
50
|
-
@child_entries ||= @args[1].map { |e| Bulkrax::Entry.find(e) }
|
51
|
-
end
|
52
|
-
|
53
|
-
def child_works_hash
|
54
|
-
@child_works_hash ||= child_entries.each_with_object({}) do |child_entry, hash|
|
55
|
-
work = child_entry.factory.find
|
56
|
-
# If we can't find the Work/Collection, raise a custom error
|
57
|
-
raise ChildWorksError if work.blank?
|
58
|
-
hash[work.id] = { class_name: work.class.to_s, entry.parser.source_identifier => child_entry.identifier }
|
59
|
-
end
|
60
|
-
end
|
61
|
-
|
62
|
-
def importer_run_id
|
63
|
-
@args[2]
|
64
|
-
end
|
65
|
-
|
66
|
-
def user
|
67
|
-
@user ||= entry.importerexporter.user
|
68
|
-
end
|
69
|
-
|
70
|
-
private
|
71
|
-
|
72
|
-
# rubocop:disable Rails/SkipsModelValidations
|
73
|
-
# Work-Collection membership is added to the child as member_of_collection_ids
|
74
|
-
# This is adding the reverse relatinship, from the child to the parent
|
75
|
-
def work_child_collection_parent(work_id)
|
76
|
-
attrs = { id: work_id, collections: [{ id: entry&.factory&.find&.id }] }
|
77
|
-
Bulkrax::ObjectFactory.new(attributes: attrs,
|
78
|
-
source_identifier_value: child_works_hash[work_id][entry.parser.source_identifier],
|
79
|
-
work_identifier: entry.parser.work_identifier,
|
80
|
-
replace_files: false,
|
81
|
-
user: user,
|
82
|
-
klass: child_works_hash[work_id][:class_name].constantize).run
|
83
|
-
ImporterRun.find(importer_run_id).increment!(:processed_children)
|
84
|
-
rescue StandardError => e
|
85
|
-
entry.status_info(e)
|
86
|
-
ImporterRun.find(importer_run_id).increment!(:failed_children)
|
87
|
-
end
|
88
|
-
|
89
|
-
# Collection-Collection membership is added to the as member_ids
|
90
|
-
def collection_parent_collection_child(member_ids)
|
91
|
-
attrs = { id: entry&.factory&.find&.id, children: member_ids }
|
92
|
-
Bulkrax::ObjectFactory.new(attributes: attrs,
|
93
|
-
source_identifier_value: entry.identifier,
|
94
|
-
work_identifier: entry.parser.work_identifier,
|
95
|
-
replace_files: false,
|
96
|
-
user: user,
|
97
|
-
klass: entry.factory_class).run
|
98
|
-
ImporterRun.find(importer_run_id).increment!(:processed_children)
|
99
|
-
rescue StandardError => e
|
100
|
-
entry.status_info(e)
|
101
|
-
ImporterRun.find(importer_run_id).increment!(:failed_children)
|
102
|
-
end
|
103
|
-
|
104
|
-
# Work-Work membership is added to the parent as member_ids
|
105
|
-
def work_parent_work_child(member_ids)
|
106
|
-
# build work_members_attributes
|
107
|
-
attrs = { id: entry&.factory&.find&.id,
|
108
|
-
work_members_attributes: member_ids.each.with_index.each_with_object({}) do |(member, index), ids|
|
109
|
-
ids[index] = { id: member }
|
110
|
-
end }
|
111
|
-
Bulkrax::ObjectFactory.new(attributes: attrs,
|
112
|
-
source_identifier_value: entry.identifier,
|
113
|
-
work_identifier: entry.parser.work_identifier,
|
114
|
-
replace_files: false,
|
115
|
-
user: user,
|
116
|
-
klass: entry.factory_class).run
|
117
|
-
ImporterRun.find(importer_run_id).increment!(:processed_children)
|
118
|
-
rescue StandardError => e
|
119
|
-
entry.status_info(e)
|
120
|
-
ImporterRun.find(importer_run_id).increment!(:failed_children)
|
121
|
-
end
|
122
|
-
# rubocop:enable Rails/SkipsModelValidations
|
123
|
-
|
124
|
-
def reschedule(entry_id, child_entry_ids, importer_run_id)
|
125
|
-
ChildRelationshipsJob.set(wait: 10.minutes).perform_later(entry_id, child_entry_ids, importer_run_id)
|
126
|
-
end
|
127
|
-
end
|
128
|
-
end
|