cdm_migrator 3.0.0 → 3.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +6 -5
- data/app/assets/stylesheets/cdm_migrator/csv_checker.css +37 -0
- data/app/controllers/cdm_migrator/application_controller.rb +5 -0
- data/app/controllers/cdm_migrator/cdm_controller.rb +4 -4
- data/app/controllers/cdm_migrator/csv_controller.rb +242 -69
- data/app/jobs/cdm_migrator/batch_create_files_job.rb +2 -3
- data/app/jobs/cdm_migrator/batch_create_works_job.rb +1 -2
- data/app/jobs/cdm_migrator/cdm_ingest_files_job.rb +4 -4
- data/app/jobs/cdm_migrator/create_work_job.rb +8 -2
- data/app/jobs/cdm_migrator/update_object_job.rb +10 -0
- data/app/models/cdm_migrator/batch_ingest.rb +7 -4
- data/app/views/cdm_migrator/csv/csv_checker.html.erb +46 -0
- data/app/views/cdm_migrator/csv/edit.html.erb +18 -0
- data/config/routes.rb +5 -2
- data/lib/cdm_migrator/version.rb +1 -1
- data/lib/generators/cdm_migrator/install/install_generator.rb +10 -2
- data/lib/generators/cdm_migrator/install/templates/config/cdm_migrator.yml +32 -19
- data/lib/generators/cdm_migrator/install/templates/sidebar/_tasks.html.erb +6 -2
- metadata +6 -3
- data/app/views/cdm_migrator/csv/file_path_checker.html.erb +0 -21
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 330afa0ea8f2de397dc822fafef651778e039b0b422ec8ab93e61bd4ed6e9ced
|
4
|
+
data.tar.gz: 6860866888bf49430fbbb4eace6e38bb7b2da09f6b8633608e7508cac374dc4c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f263a7da82234a60ac6ac4f73529b226762b483dee9ce093712f8dd3c34dc97b3b1b96b5b62cbc98d0de5f2dc1157367b5c1352a5cb79bd7703f45a707ceb034
|
7
|
+
data.tar.gz: f4f095574c88c63970d2de9032ad96ad134ff34952b78068a2ddceee404d9998134ccfac15ee29ca75d2ac9c7b8f4a3a4db33030b6218d8b100afa64ea357cd3
|
data/README.md
CHANGED
@@ -27,11 +27,12 @@ to insert the yml and add a link to your Hyrax dashboard
|
|
27
27
|
|
28
28
|
## Usage
|
29
29
|
1. Add your ContentDM url and api port to the cdm_migrator.yml file.
|
30
|
-
2.
|
31
|
-
3.
|
32
|
-
4.
|
33
|
-
5.
|
34
|
-
6.
|
30
|
+
2. Configure the CSV Checker with the appropriate fields, paths, or multi-value separator (in cdm_migrator.yml).
|
31
|
+
3. Navigate to the *cdm_migrator/collection* url to select your contentdm collection and what type of work you want to export it to and click "choose mappings".
|
32
|
+
4. Map the ContentDM fields to your Hyrax work and file fields\* and click "generate CSV".
|
33
|
+
5. Refine the CSV as you see fit. Optional: go to *cdm_migrator/csv_checker* and upload it to validate metadata fields and/or file paths.
|
34
|
+
6. Navigate to the *cdm_migrator/upload* url; choose your multi-value separator (default is |) and upload your CSV file.
|
35
|
+
7. Done.
|
35
36
|
|
36
37
|
\* cdm_migrator uses the generated Hyrax forms (ex. Hyrax::Forms::GenericWorkForm) in your host application to obtain it's terms for mapping. If you have added terms to your FileSet model extend the Hyrax::Forms::FileSetEditForm with Hyrax::FileSetForm in your host application so that the changes will be detected by the migrator. You can also add a list of fields in the yml file, under "default fields".
|
37
38
|
|
@@ -0,0 +1,37 @@
|
|
1
|
+
.csv-collapse-link.collapsed {
|
2
|
+
display: inline-block;
|
3
|
+
vertical-align: top;
|
4
|
+
}
|
5
|
+
|
6
|
+
#csv-collapse-link-text::after {
|
7
|
+
content: "❯";
|
8
|
+
display: inline-block;
|
9
|
+
font-size: smaller;
|
10
|
+
right: 15px;
|
11
|
+
transform: rotate(90deg);
|
12
|
+
margin-left: 0.5em;
|
13
|
+
}
|
14
|
+
|
15
|
+
.collapsed #csv-collapse-link-text::after {
|
16
|
+
transform: rotate(0deg);
|
17
|
+
transition: transform 0.1s ease;
|
18
|
+
margin-left: 2px;
|
19
|
+
vertical-align: top;
|
20
|
+
}
|
21
|
+
|
22
|
+
#errors-explanation ul li {
|
23
|
+
list-style-type: none;
|
24
|
+
margin-bottom: 2px;
|
25
|
+
}
|
26
|
+
|
27
|
+
#errors-explanation ul {
|
28
|
+
padding-left: 0;
|
29
|
+
}
|
30
|
+
|
31
|
+
#errors-explanation {
|
32
|
+
margin-top: 5px;
|
33
|
+
}
|
34
|
+
|
35
|
+
#csv-form {
|
36
|
+
margin-top: 1em;
|
37
|
+
}
|
@@ -104,10 +104,10 @@ module CdmMigrator
|
|
104
104
|
|
105
105
|
def load_yaml
|
106
106
|
stripped_url = request.base_url.dup.gsub(/https?:\/\//, '').gsub(/:[0-9]*/,'')
|
107
|
-
if CdmMigrator::Engine.config['
|
108
|
-
tenant = CdmMigrator::Engine.config['
|
107
|
+
if CdmMigrator::Engine.config['tenant_settings'].key? stripped_url
|
108
|
+
tenant = CdmMigrator::Engine.config['tenant_settings'][stripped_url]['cdm_api']
|
109
109
|
else
|
110
|
-
tenant = CdmMigrator::Engine.config['
|
110
|
+
tenant = CdmMigrator::Engine.config['tenant_settings']['default']['cdm_api']
|
111
111
|
end
|
112
112
|
@cdm_url = tenant['url']
|
113
113
|
@cdm_port = tenant['port']
|
@@ -121,7 +121,7 @@ module CdmMigrator
|
|
121
121
|
filename = child ? child['pagefile'] : "#{rec.first}.#{rec.last}"
|
122
122
|
|
123
123
|
if params[:file_system]=='true'
|
124
|
-
"file://#{file_path(
|
124
|
+
"file://#{file_path(cisoptr)}"
|
125
125
|
elsif @cdm_api == 'server'
|
126
126
|
"#{@cdm_url}:#{@cdm_port}/cgi-bin/showfile.exe?CISOROOT=#{params[:collection]}&CISOPTR=#{cisoptr}"
|
127
127
|
else
|
@@ -1,20 +1,20 @@
|
|
1
1
|
module CdmMigrator
|
2
|
-
|
3
|
-
|
2
|
+
class CsvController < ApplicationController
|
3
|
+
helper_method :default_page_title, :admin_host?, :available_translations, :available_works
|
4
4
|
include ActionView::Helpers::UrlHelper
|
5
|
-
|
5
|
+
layout 'hyrax/dashboard' if Hyrax
|
6
6
|
before_action :authenticate, except: :index
|
7
|
+
before_action :load_config, only: :csv_checker
|
7
8
|
|
8
|
-
def
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
end
|
9
|
+
def csv_checker
|
10
|
+
if params[:file]
|
11
|
+
check_csv params[:file].path
|
12
|
+
if @error_list.blank?
|
13
|
+
flash[:notice] = "All data are valid."
|
14
|
+
else
|
15
|
+
flash[:error] = "The CSV Checker found some errors in the CSV. Please correct them and check again."
|
16
|
+
end
|
17
|
+
end
|
18
18
|
end
|
19
19
|
|
20
20
|
def index
|
@@ -27,34 +27,34 @@ module CdmMigrator
|
|
27
27
|
end
|
28
28
|
end
|
29
29
|
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
30
|
+
def upload
|
31
|
+
@admin_sets = AdminSet.all.map { |as| [as.title.first, as.id] }
|
32
|
+
@collections = Collection.all.map { |col| [col.title.first, col.id] }
|
33
|
+
end
|
34
34
|
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
csv
|
41
|
-
|
42
|
-
|
35
|
+
def create
|
36
|
+
dir = Rails.root.join('public', 'uploads', 'csvs')
|
37
|
+
FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
|
38
|
+
time = DateTime.now.strftime('%s')
|
39
|
+
filename = params[:csv_import][:csv_file].original_filename.gsub('.csv', "#{time}.csv")
|
40
|
+
csv = dir.join(filename).to_s
|
41
|
+
File.open(csv, 'wb') do |file|
|
42
|
+
file.write(params[:csv_import][:csv_file].read)
|
43
43
|
end
|
44
|
-
|
45
|
-
if @
|
46
|
-
flash[:error] = "some
|
44
|
+
check_csv csv
|
45
|
+
if @error_list.present?
|
46
|
+
flash[:error] = "Cdm Migrator found some problems with the CSV. Use the CSV Checker for more details."
|
47
47
|
end
|
48
48
|
parse_csv(csv, params[:csv_import][:mvs])
|
49
49
|
|
50
50
|
ingest = BatchIngest.new({
|
51
|
-
data:
|
52
|
-
size:
|
53
|
-
csv:
|
54
|
-
admin_set_id:
|
51
|
+
data: @works,
|
52
|
+
size: @works.length,
|
53
|
+
csv: csv,
|
54
|
+
admin_set_id: params[:admin_set],
|
55
55
|
collection_id: params[:collection],
|
56
|
-
user_id:
|
57
|
-
message:
|
56
|
+
user_id: current_user.id,
|
57
|
+
message: @path_list.blank? ? nil : @path_list.to_s.gsub("\"", """)
|
58
58
|
})
|
59
59
|
if ingest.save! && @path_list.blank?
|
60
60
|
BatchCreateWorksJob.perform_later(ingest, current_user)
|
@@ -75,9 +75,8 @@ module CdmMigrator
|
|
75
75
|
end
|
76
76
|
|
77
77
|
def generate
|
78
|
-
headers =
|
79
|
-
skip
|
80
|
-
"embargo_id", "lease_id", "access_control_id", "representative_id"]
|
78
|
+
headers = %w(type url)
|
79
|
+
skip = %w(id head tail depositor date_uploaded date_modified import_url thumbnail_id embargo_id lease_id access_control_id representative_id)
|
81
80
|
GenericWork.new.attributes.each do |key, val|
|
82
81
|
headers << "work_#{key}" unless skip.include? key
|
83
82
|
end
|
@@ -87,19 +86,108 @@ module CdmMigrator
|
|
87
86
|
fname = "template_#{DateTime.now.to_i}"
|
88
87
|
render plain: CSV.generate { |csv| csv << headers }, content_type: 'text/csv'
|
89
88
|
end
|
90
|
-
|
91
|
-
|
89
|
+
|
90
|
+
def edit
|
91
|
+
@collections = ::Collection.all.map { |c| [c.title.first, c.id] }
|
92
|
+
end
|
93
|
+
|
94
|
+
def update
|
95
|
+
mvs = params[:csv_update][:mvs]
|
96
|
+
csv = CSV.parse(params[:csv_update][:csv_file].read, headers: true, encoding: 'utf-8').map(&:to_hash)
|
97
|
+
csv.each do |row|
|
98
|
+
obj = ActiveFedora::Base.find row['id']
|
99
|
+
type = row.first.last
|
100
|
+
if type.nil?
|
101
|
+
next
|
102
|
+
elsif type.include? "Work"
|
103
|
+
metadata = create_data(row.except('id', 'type'), work_form(type), obj, mvs)
|
104
|
+
elsif type.include? "File"
|
105
|
+
metadata = create_data(row.except('id', 'type'), work_form(type), obj, mvs)
|
106
|
+
end
|
107
|
+
unless metadata.nil?
|
108
|
+
obj.attributes = metadata
|
109
|
+
obj.save
|
110
|
+
end
|
111
|
+
end
|
112
|
+
flash[:notice] = "csv successfully uploaded"
|
113
|
+
redirect_to csv_edit_path
|
114
|
+
end
|
115
|
+
|
116
|
+
def export
|
117
|
+
solr = RSolr.connect url: Account.find_by(tenant: Apartment::Tenant.current).solr_endpoint.url
|
118
|
+
response = solr.get 'select', params: {
|
119
|
+
q: "member_of_collection_ids_ssim:#{params[:collection_id]}",
|
120
|
+
fl: "id"
|
121
|
+
}
|
122
|
+
unless response['response']['docs'].empty? || response['response']['docs'][0].empty?
|
123
|
+
work_ids = response['response']['docs'].map { |doc| doc['id'] }
|
124
|
+
end
|
125
|
+
#works = ::ActiveFedora::Base.where member_of_collection_ids_ssim: params[:collection_id]
|
126
|
+
@csv_headers = ['type'] + work_fields
|
127
|
+
@csv_array = [@csv_headers.join(',')]
|
128
|
+
work_ids.each do |work_id|
|
129
|
+
doc = ::SolrDocument.find work_id
|
130
|
+
add_line doc
|
131
|
+
doc._source[:file_set_ids_ssim].each do |file_id|
|
132
|
+
file_doc = ::SolrDocument.find file_id
|
133
|
+
add_line file_doc
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
send_data @csv_array.join("\n"),
|
138
|
+
:type => 'text/csv; charset=iso-8859-5; header=present',
|
139
|
+
:disposition => "attachment; filename=export.csv"
|
140
|
+
end
|
141
|
+
|
142
|
+
private
|
92
143
|
|
93
144
|
def authenticate
|
94
145
|
authorize! :create, available_works.first
|
95
146
|
end
|
96
147
|
|
148
|
+
def add_line doc
|
149
|
+
line_hash = {}
|
150
|
+
line_hash['type'] = doc._source[:has_model_ssim].first
|
151
|
+
work_fields.each do |field|
|
152
|
+
line_hash[field] = create_cell doc, field
|
153
|
+
end
|
154
|
+
@csv_array << line_hash.values_at(*@csv_headers).map { |cell| cell = '' if cell.nil?; "\"#{cell.gsub("\"", "\"\"")}\"" }.join(',')
|
155
|
+
|
156
|
+
end
|
157
|
+
|
158
|
+
def work_fields
|
159
|
+
@fields ||= available_works.map { |work| work.new.attributes.keys }.flatten.uniq - excluded_fields
|
160
|
+
end
|
161
|
+
|
162
|
+
def excluded_fields
|
163
|
+
%w[date_uploaded date_modified head tail state proxy_depositor on_behalf_of arkivo_checksum label
|
164
|
+
relative_path import_url part_of resource_type access_control_id
|
165
|
+
representative_id thumbnail_id rendering_ids admin_set_id embargo_id
|
166
|
+
lease_id]
|
167
|
+
end
|
168
|
+
|
169
|
+
def create_cell w, field
|
170
|
+
if field.include? 'date'
|
171
|
+
if w._source[field+'_tesim'].is_a?(Array)
|
172
|
+
w._source[field+'_tesim'].join('|')
|
173
|
+
else
|
174
|
+
w._source[field+'_tesim']
|
175
|
+
end
|
176
|
+
elsif w.respond_to?(field.to_sym)
|
177
|
+
if w.send(field).is_a?(Array)
|
178
|
+
w.send(field).join('|')
|
179
|
+
else
|
180
|
+
w.send(field)
|
181
|
+
end
|
182
|
+
end
|
183
|
+
end
|
184
|
+
|
97
185
|
def available_works
|
98
186
|
@available_works ||= Hyrax::QuickClassificationQuery.new(current_user).authorized_models
|
99
187
|
end
|
100
188
|
|
101
189
|
def parse_csv csv, mvs
|
102
|
-
csv
|
190
|
+
csv = CSV.parse(File.read(csv), headers: true, encoding: 'utf-8').map(&:to_hash)
|
103
191
|
@works = []
|
104
192
|
csv.each do |row|
|
105
193
|
type = row.first.last
|
@@ -115,19 +203,104 @@ module CdmMigrator
|
|
115
203
|
end
|
116
204
|
end
|
117
205
|
|
118
|
-
|
119
|
-
|
120
|
-
|
206
|
+
def load_config
|
207
|
+
tenant = Account.find_by(tenant: Apartment::Tenant.current).cname
|
208
|
+
if CdmMigrator::Engine.config['tenant_settings'].has_key?(tenant)
|
209
|
+
settings = CdmMigrator::Engine.config['tenant_settings'][tenant]['csv_checker']
|
210
|
+
if settings.present?
|
211
|
+
# .map will throw an error if settings[key] has no value
|
212
|
+
@edtf_fields = settings['edtf_fields'].map(&:to_sym) if settings['edtf_fields']
|
213
|
+
@uri_fields = settings['valid_uri_fields'].map(&:to_sym) if settings['valid_uri_fields']
|
214
|
+
@separator = settings['multi_value_separator']
|
215
|
+
@separator_fields = settings['separator_fields'].map(&:to_sym) if settings['separator_fields']
|
216
|
+
@path_to_drive = settings['path_to_drive']
|
217
|
+
else
|
218
|
+
raise "Cdm Migrator couldn't find any configured settings. Are they in cdm_migrator.yml?"
|
219
|
+
end
|
220
|
+
else
|
221
|
+
raise "Cdm Migrator couldn't find this tenant. Is it configured?"
|
222
|
+
end
|
223
|
+
end
|
224
|
+
|
225
|
+
def check_csv csv_file
|
226
|
+
row_number = 1
|
227
|
+
@error_list = {}
|
228
|
+
check_mounted_drive if @path_to_drive.present?
|
229
|
+
|
230
|
+
CSV.foreach(csv_file, headers: true, header_converters: :symbol) do |row|
|
231
|
+
row_number +=1 # Tells user what CSV row the error is on
|
232
|
+
if row[:object_type].include? "Work"
|
233
|
+
check_edtf(row_number, row) if @edtf_fields.present?
|
234
|
+
check_uris(row_number, row) if @uri_fields.present?
|
235
|
+
if params[:multi_value_separator].present? and @separator_fields.present?
|
236
|
+
check_separator(row_number, row, params[:multi_value_separator])
|
237
|
+
else
|
238
|
+
alert_message = "No multi-value separator character was selected or no fields were configured. CSV Checker didn't check for valid separators."
|
239
|
+
if flash[:alert] and flash[:alert].exclude?(alert_message) # Only add this message once, rather than per line
|
240
|
+
flash[:alert] << alert_message
|
241
|
+
elsif flash[:alert].blank?
|
242
|
+
flash[:alert] = Array.wrap(alert_message)
|
243
|
+
end
|
244
|
+
end
|
245
|
+
elsif row[:object_type] == "File"
|
246
|
+
check_file_path(row_number, row[:url])
|
247
|
+
else
|
248
|
+
@error_list[row_number] = { "object_type" => "No or unknown object type. Please give a valid type (e.g. GenericWork, File)." }
|
249
|
+
end
|
250
|
+
@error_list.delete_if { |key, value| value.blank? } # Data are valid, no need to print the row
|
251
|
+
end
|
252
|
+
end
|
253
|
+
|
254
|
+
def check_mounted_drive
|
255
|
+
drive_address = @path_to_drive
|
256
|
+
unless Dir.exist?(drive_address) and !Dir[drive_address].empty?
|
257
|
+
flash[:alert] = "CSV Checker can't find the mounted drive to check file paths, so some paths may be mislabelled as incorrect. Please contact the administrator or try again later."
|
258
|
+
end
|
259
|
+
end
|
260
|
+
|
261
|
+
def check_file_path(row_number, file_path)
|
262
|
+
if file_path.nil?
|
263
|
+
@error_list[row_number] = { "url" => "url is blank." }
|
264
|
+
elsif File.file?(file_path.gsub("file://", "")) == false
|
265
|
+
@error_list[row_number] = { "url" => "No file found at #{file_path}" }
|
266
|
+
end
|
267
|
+
end
|
268
|
+
|
269
|
+
def check_edtf(row_number, row)
|
270
|
+
edtf_fields = @edtf_fields
|
271
|
+
edtf_errors = edtf_fields.each_with_object({}) do |field, hash|
|
272
|
+
if Date.edtf(row[field]) == nil and row[field] != "unknown"
|
273
|
+
hash[field.to_s] = "Blank or not a valid EDTF date."
|
274
|
+
end
|
275
|
+
end
|
276
|
+
@error_list[row_number] = edtf_errors
|
277
|
+
end
|
278
|
+
|
279
|
+
# <Example: should be http://rightsstatements.org/vocab/etc. NOT https://rightsstatements.org/page/etc.
|
280
|
+
def check_uris(row_number, row)
|
281
|
+
uri_fields = @uri_fields
|
282
|
+
uri_errors = uri_fields.each_with_object({}) do |field, hash|
|
283
|
+
if row[field].include? "page"
|
284
|
+
hash[field.to_s] = "Links to page instead of URI. (e.g. https://rightsstatements.org/page/etc. instead of http://rightsstatements.org/vocab/etc.)"
|
285
|
+
end
|
286
|
+
end
|
287
|
+
@error_list[row_number].merge!(uri_errors)
|
288
|
+
end
|
121
289
|
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
290
|
+
# Check multi-value separators
|
291
|
+
def check_separator(row_number, row, character)
|
292
|
+
uri_fields = @separator_fields
|
293
|
+
separator_errors = uri_fields.each_with_object({}) do |field, hash|
|
294
|
+
value = row[field]
|
295
|
+
if value.present?
|
296
|
+
URI.extract(value).each { |uri| value.gsub!(uri, '') }
|
297
|
+
unless value.split("").all?(character) # Check if remaining characters are the correct separator
|
298
|
+
hash[field.to_s] = "May contain the wrong multi-value separator (i.e. not #{character})."
|
128
299
|
end
|
129
300
|
end
|
130
301
|
end
|
302
|
+
@error_list[row_number].merge!(separator_errors)
|
303
|
+
end
|
131
304
|
|
132
305
|
def default_page_title
|
133
306
|
'CSV Batch Uploader'
|
@@ -139,8 +312,8 @@ module CdmMigrator
|
|
139
312
|
|
140
313
|
def available_translations
|
141
314
|
{
|
142
|
-
|
143
|
-
|
315
|
+
'en' => 'English',
|
316
|
+
'fr' => 'French'
|
144
317
|
}
|
145
318
|
end
|
146
319
|
|
@@ -162,12 +335,12 @@ module CdmMigrator
|
|
162
335
|
end
|
163
336
|
|
164
337
|
def create_data data, type, object, mvs
|
165
|
-
final_data
|
338
|
+
final_data = {}
|
166
339
|
accepted_terms = type.required_fields + secondary_terms(type)
|
167
340
|
data.each do |key, att|
|
168
|
-
if(att.nil? || att.empty? || key.to_s.include?("object_type") || !accepted_terms.include?(key.to_sym)
|
341
|
+
if (att.nil? || att.empty? || key.to_s.include?("object_type") || !accepted_terms.include?(key.to_sym))
|
169
342
|
next
|
170
|
-
elsif(object.send(key).nil?)
|
343
|
+
elsif (object.send(key).nil?)
|
171
344
|
final_data[key] = att
|
172
345
|
else
|
173
346
|
final_data[key] = att.split(mvs)
|
@@ -176,23 +349,23 @@ module CdmMigrator
|
|
176
349
|
final_data
|
177
350
|
end
|
178
351
|
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
352
|
+
def create_lease visibility, status_after, date
|
353
|
+
lease = Hydra::AccessControls::Lease.new(visibility_during_lease: visibility,
|
354
|
+
visibility_after_lease: status_after, lease_expiration_date: @lease_date)
|
355
|
+
lease.save
|
356
|
+
end
|
184
357
|
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
358
|
+
def create_embargo visibility
|
359
|
+
embargo = Hydra::AccessControls::Embargo.new
|
360
|
+
embargo.visibility_during_embargo = visibility
|
361
|
+
embargo.visibility_after_embargo = @status_after
|
362
|
+
embargo.embargo_release_date = @embargo_date
|
363
|
+
embargo.save
|
364
|
+
end
|
192
365
|
|
193
366
|
def log(user)
|
194
|
-
|
195
|
-
|
367
|
+
Hyrax::Operation.create!(user: user,
|
368
|
+
operation_type: "Attach Remote File")
|
196
369
|
end
|
197
|
-
|
370
|
+
end
|
198
371
|
end
|
@@ -2,7 +2,7 @@ module CdmMigrator
|
|
2
2
|
class BatchCreateFilesJob < ActiveJob::Base
|
3
3
|
queue_as Hyrax.config.ingest_queue_name
|
4
4
|
|
5
|
-
def perform work, ingest_work, user
|
5
|
+
def perform work, ingest_work, user
|
6
6
|
ingest_work.files.each do |file|
|
7
7
|
url = file[:url]
|
8
8
|
last_file = ingest_work.files.last==file
|
@@ -11,10 +11,9 @@ module CdmMigrator
|
|
11
11
|
actor = Hyrax::Actors::FileSetActor.new(fs, user)
|
12
12
|
actor.create_metadata#(work, visibility: work.visibility)
|
13
13
|
actor.attach_file_to_work(work)
|
14
|
-
#byebug
|
15
14
|
fs.attributes = file[:metadata]
|
16
15
|
fs.save!
|
17
|
-
CdmIngestFilesJob.perform_later(fs, url, user, ingest_work, last_file
|
16
|
+
CdmIngestFilesJob.perform_later(fs, url, user, ingest_work, last_file)
|
18
17
|
end
|
19
18
|
end
|
20
19
|
end
|
@@ -4,10 +4,9 @@ module CdmMigrator
|
|
4
4
|
|
5
5
|
def perform(ingest, user)
|
6
6
|
ingest.data.each do |w|
|
7
|
-
last_work = ingest.data.last==w
|
8
7
|
ingest_work = IngestWork.new(w, ingest.id)
|
9
8
|
ingest_work.save!
|
10
|
-
CreateWorkJob.perform_later ingest_work, user, ingest.admin_set_id, ingest.collection_id
|
9
|
+
CreateWorkJob.perform_later ingest_work, user, ingest.admin_set_id, ingest.collection_id
|
11
10
|
end
|
12
11
|
|
13
12
|
end
|
@@ -2,7 +2,7 @@ module CdmMigrator
|
|
2
2
|
class CdmIngestFilesJob < ActiveJob::Base
|
3
3
|
queue_as Hyrax.config.ingest_queue_name
|
4
4
|
|
5
|
-
def perform(fs, url, user, ingest_work = nil, last_file = false
|
5
|
+
def perform(fs, url, user, ingest_work = nil, last_file = false)
|
6
6
|
if url.include?("http") && File.extname(url).include?("pdf")
|
7
7
|
download = open(url)
|
8
8
|
dir = Rails.root.join('public', 'uploads', 'csv_pdfs')
|
@@ -16,14 +16,14 @@ module CdmMigrator
|
|
16
16
|
IO.copy_stream(download, url)
|
17
17
|
url = "file://"+url.to_s
|
18
18
|
end
|
19
|
-
uri = URI.parse(url.gsub(' ','%20'))
|
19
|
+
uri = URI.parse(url.gsub(' ','%20').gsub(/[\[\]@#\$\*{}]/, ""))
|
20
20
|
if uri.scheme == 'file'
|
21
|
-
IngestLocalFileJob.perform_now(fs,
|
21
|
+
IngestLocalFileJob.perform_now(fs, url.gsub('file://',''), user)
|
22
22
|
else
|
23
|
+
URI.parse(url.gsub(' ','%20'))
|
23
24
|
ImportUrlJob.perform_now(fs, log(user))
|
24
25
|
end
|
25
26
|
ingest_work.update_attribute('complete', true) if last_file
|
26
|
-
BatchIngest.find(ingest_work.id).update_attribute('complete', true) if last_work
|
27
27
|
end
|
28
28
|
|
29
29
|
def log(user)
|
@@ -2,18 +2,24 @@ module CdmMigrator
|
|
2
2
|
class CreateWorkJob < ActiveJob::Base
|
3
3
|
queue_as Hyrax.config.ingest_queue_name
|
4
4
|
|
5
|
-
def perform(ingest_work, user, admin_set_id, collection_id
|
5
|
+
def perform(ingest_work, user, admin_set_id, collection_id)
|
6
6
|
admin_set = ::AdminSet.find(admin_set_id) rescue nil
|
7
7
|
collection = ::Collection.find(collection_id) rescue nil
|
8
8
|
work = Object.const_get(ingest_work.work_type).new
|
9
9
|
#status_after, embargo_date, lease_date = nil, nil, nil
|
10
10
|
work.apply_depositor_metadata(user)
|
11
11
|
work.attributes = ingest_work.data
|
12
|
+
if ingest_work.data.has_key? 'downloadable'
|
13
|
+
# Convert string to boolean
|
14
|
+
work.downloadable = ActiveModel::Type::Boolean.new.cast(ingest_work.data['downloadable'])
|
15
|
+
elsif work.attributes.include? 'downloadable' # Set work to downloadable by default
|
16
|
+
work.downloadable = true
|
17
|
+
end
|
12
18
|
work.member_of_collections = [collection] if collection
|
13
19
|
work.admin_set = admin_set if admin_set
|
14
20
|
work.date_uploaded = DateTime.now
|
15
21
|
work.save
|
16
|
-
BatchCreateFilesJob.perform_later work, ingest_work, user
|
22
|
+
BatchCreateFilesJob.perform_later work, ingest_work, user
|
17
23
|
|
18
24
|
end
|
19
25
|
end
|
@@ -7,11 +7,14 @@ module CdmMigrator
|
|
7
7
|
end
|
8
8
|
|
9
9
|
def progress
|
10
|
-
if complete?
|
10
|
+
return "Complete" if complete?
|
11
|
+
completed = IngestWork.where(batch_ingest_id: id, complete: true ).length
|
12
|
+
if completed==data.length
|
13
|
+
complete=true
|
14
|
+
save
|
11
15
|
"Complete"
|
12
16
|
else
|
13
|
-
completed
|
14
|
-
"#{completed}/#{size}"
|
17
|
+
"#{completed.to_s}/#{size}"
|
15
18
|
end
|
16
19
|
end
|
17
20
|
|
@@ -20,7 +23,7 @@ module CdmMigrator
|
|
20
23
|
end
|
21
24
|
|
22
25
|
def complete?
|
23
|
-
|
26
|
+
complete
|
24
27
|
end
|
25
28
|
|
26
29
|
def message?
|
@@ -0,0 +1,46 @@
|
|
1
|
+
<% provide :page_title, "CSV Checker" %>
|
2
|
+
|
3
|
+
<%#= flash[:alert].join("<br/>") if flash[:alert] %>
|
4
|
+
|
5
|
+
<h1>CSV Checker</h1>
|
6
|
+
|
7
|
+
<!-- <div class="row"> -->
|
8
|
+
<p>This tool validates CSV data and creates a table listing any errors.</p>
|
9
|
+
|
10
|
+
<a role="button" class="collapse-toggle collapsed csv-collapse-link" data-toggle="collapse" data-target="#errors-explanation" aria-expanded="false">
|
11
|
+
<span id="csv-collapse-link-text">Expand for more details</span>
|
12
|
+
</a>
|
13
|
+
<div class="collapse" id="errors-explanation">
|
14
|
+
<ul>
|
15
|
+
<li><strong>File paths: </strong>The url field contains a valid path to a file.</li>
|
16
|
+
<li><strong>Multi-value separator: </strong>Configured fields with URIs contain the right separator character (e.g. |).</li>
|
17
|
+
<% if @path_to_drive.present? %>
|
18
|
+
<li><strong>Mounted drive: </strong>A mounted directory (folder) exists and is not empty.</li>
|
19
|
+
<% end %>
|
20
|
+
<% if @edtf_fields.present? %>
|
21
|
+
<li><strong>EDTF dates: </strong>Configured fields contain valid <a href="https://www.loc.gov/standards/datetime/" target="_blank">EDTF</a> dates or "unknown."</li>
|
22
|
+
<% end %>
|
23
|
+
<% if @uri_fields.present? %>
|
24
|
+
<li><strong>Valid URIs ("page" vs "vocab"): </strong>Configured fields with URIs link to the "vocab" address rather than the "page" address. For example, rights_statement should be "http://rightsstatement.org/vocab/..." and not "https://rightsstatement.org/page/..."</li>
|
25
|
+
<% end %>
|
26
|
+
</ul>
|
27
|
+
</div>
|
28
|
+
|
29
|
+
|
30
|
+
|
31
|
+
<%= form_tag(check_csv_path, remote: true, method: :post, multipart: true, id: "csv-form") do %>
|
32
|
+
<div class="input-group">
|
33
|
+
<%= label_tag :multi_value_separator %>
|
34
|
+
<%= text_field_tag(:multi_value_separator, @separator, size: 1) %>
|
35
|
+
<%= file_field_tag(:file, class: "form-control-file") %>
|
36
|
+
<%= hidden_field_tag :authenticity_token, value: form_authenticity_token %>
|
37
|
+
<%= button_tag(type: :submit, class: "btn btn-large btn-primary", style: "margin-top: 1em;") do %>
|
38
|
+
Check CSV
|
39
|
+
<% end %>
|
40
|
+
<% end %>
|
41
|
+
</div>
|
42
|
+
<!-- </div> -->
|
43
|
+
|
44
|
+
<div id="error_list" class="col-md-10 offset-md-1">
|
45
|
+
<%= render 'error_list' %>
|
46
|
+
</div>
|
@@ -0,0 +1,18 @@
|
|
1
|
+
|
2
|
+
<h1><span class="fa fa-map"></span> Export </h1>
|
3
|
+
|
4
|
+
<%= form_tag main_app.csv_export_path, method: :post do %>
|
5
|
+
<%= select_tag 'collection_id', options_for_select(@collections), include_blank: true %>
|
6
|
+
<%= submit_tag "Download CSV", class: 'btn btn-primary' %>
|
7
|
+
<% end %>
|
8
|
+
|
9
|
+
<h1><span class="fa fa-map"></span> Update </h1>
|
10
|
+
|
11
|
+
<%= form_for :csv_update, url: csv_update_path do |f| %>
|
12
|
+
<%= f.label 'Multi-value Separator:' %>
|
13
|
+
<%= f.text_field 'mvs' %>
|
14
|
+
<br />
|
15
|
+
<%= f.file_field 'csv_file' %>
|
16
|
+
<br />
|
17
|
+
<%= f.submit 'Save' %>
|
18
|
+
<% end %>
|
data/config/routes.rb
CHANGED
@@ -5,9 +5,12 @@ Rails.application.routes.draw do
|
|
5
5
|
get '/cdm_migrator/my/batches', to: 'cdm_migrator/csv#index', as: 'csv_my_batches'
|
6
6
|
get '/cdm_migrator/batches', to: 'cdm_migrator/csv#index', as: 'csv_all_batches'
|
7
7
|
get '/cdm_migrator/rerun/:id', to: 'cdm_migrator/csv#rerun', as: 'csv_rerun'
|
8
|
+
get '/cdm_migrator/edit', to: 'cdm_migrator/csv#edit', as: 'csv_edit'
|
9
|
+
post '/cdm_migrator/export', to: 'cdm_migrator/csv#export', as: 'csv_export'
|
10
|
+
post '/cdm_migrator/update', to: 'cdm_migrator/csv#update', as: 'csv_update'
|
8
11
|
|
9
|
-
get '/cdm_migrator/
|
10
|
-
post '/cdm_migrator/
|
12
|
+
get '/cdm_migrator/csv_checker', to: 'cdm_migrator/csv#csv_checker', as: 'csv_checker'
|
13
|
+
post '/cdm_migrator/csv_checker', to: 'cdm_migrator/csv#csv_checker', as: 'check_csv'
|
11
14
|
|
12
15
|
get '/cdm_migrator/collection', to: 'cdm_migrator/cdm#collection', as: 'cdm_start'
|
13
16
|
post '/cdm_migrator/mappings/', to: 'cdm_migrator/cdm#mappings', as: 'cdm_mappings'
|
data/lib/cdm_migrator/version.rb
CHANGED
@@ -22,8 +22,8 @@ class CdmMigrator::InstallGenerator < Rails::Generators::Base
|
|
22
22
|
" <%= menu.nav_link(main_app.cdm_start_path) do %>\n" \
|
23
23
|
" <span class=\"fa fa-map\"></span> <span class=\"sidebar-action-text\"><%= t('CDM Mapping Tool') %></span>\n" \
|
24
24
|
" <% end %>\n" \
|
25
|
-
" <%= menu.nav_link(main_app.
|
26
|
-
" <span class=\"fa fa-check-circle\"></span><span>
|
25
|
+
" <%= menu.nav_link(main_app.csv_checker_path) do %>\n" \
|
26
|
+
" <span class=\"fa fa-check-circle\"></span><span>CSV Checker</span>\n" \
|
27
27
|
" <% end %>\n" \
|
28
28
|
" <%= menu.nav_link(main_app.csv_upload_path) do %>\n"\
|
29
29
|
" <span class=\"fa fa-angle-double-up\"></span> <span class=\"sidebar-action-text\"><%= t('CSV Batch Uploader') %></span>\n" \
|
@@ -67,5 +67,13 @@ class CdmMigrator::InstallGenerator < Rails::Generators::Base
|
|
67
67
|
def inject_content_dm_yml
|
68
68
|
copy_file("config/cdm_migrator.yml", "config/cdm_migrator.yml") unless File.file?("config/cdm_migrator.yml")
|
69
69
|
end
|
70
|
+
|
71
|
+
def inject_stylesheets
|
72
|
+
css_file_path = "app/assets/stylesheets/application.css"
|
73
|
+
copy_file("stylesheets/csv_checker.css", "app/assets/stylesheets/csv_checker.css") unless File.file?("app/assets/styelsheets/csv_checker.css")
|
74
|
+
insert_into_file css_file_path, :before => " *= require_self\n" do
|
75
|
+
" *= require csv_checker\n "
|
76
|
+
end
|
77
|
+
end
|
70
78
|
|
71
79
|
end
|
@@ -1,25 +1,38 @@
|
|
1
|
-
|
1
|
+
tenant_settings:
|
2
2
|
tenant1.institution.com:
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
3
|
+
cdm_api:
|
4
|
+
url: 'http://your-content-dm-host'
|
5
|
+
port: 8080
|
6
|
+
type: 'front'
|
7
|
+
dirs:
|
8
|
+
dir1: '/dir1/path/goes/here'
|
9
|
+
dir2: '/dir2/path/goes/here'
|
9
10
|
tenant2.institution.com:
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
11
|
+
cdm_api:
|
12
|
+
url: 'http://your-content-dm-host'
|
13
|
+
port: 8080
|
14
|
+
type: 'front'
|
15
|
+
dirs:
|
16
|
+
dir1: '/dir1/path/goes/here'
|
17
|
+
dir2: '/dir2/path/goes/here'
|
16
18
|
default:
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
19
|
+
cdm_api:
|
20
|
+
url: 'http://your-content-dm-host'
|
21
|
+
port: 8080
|
22
|
+
type: 'front'
|
23
|
+
dirs:
|
24
|
+
dir1: '/dir1/path/goes/here'
|
25
|
+
dir2: '/dir2/path/goes/here'
|
26
|
+
csv_checker:
|
27
|
+
edtf_fields:
|
28
|
+
# - date_created
|
29
|
+
valid_uri_fields:
|
30
|
+
# - rights_statement
|
31
|
+
# - genre
|
32
|
+
separator_fields:
|
33
|
+
# - subject
|
34
|
+
multi_value_separator: # '|'
|
35
|
+
path_to_drive: #'/mnt/drive'
|
23
36
|
default_fields:
|
24
37
|
- title
|
25
38
|
- creator
|
@@ -8,8 +8,8 @@
|
|
8
8
|
<span class="fa fa-map"></span> <span class="sidebar-action-text"><%= t('CDM Mapping Tool') %></span>
|
9
9
|
<% end %>
|
10
10
|
|
11
|
-
<%= menu.nav_link(main_app.
|
12
|
-
<span class="fa fa-check-circle"></span><span class="sidebar-action-text"><%= t('
|
11
|
+
<%= menu.nav_link(main_app.csv_checker_path) do %>
|
12
|
+
<span class="fa fa-check-circle"></span><span class="sidebar-action-text"><%= t('CSV Checker') %></span>
|
13
13
|
<% end %>
|
14
14
|
|
15
15
|
<%= menu.nav_link(main_app.csv_upload_path) do %>
|
@@ -19,6 +19,10 @@
|
|
19
19
|
<%= menu.nav_link(main_app.csv_my_batches_path) do %>
|
20
20
|
<span class="fa fa-database"></span> <span class="sidebar-action-text"><%= t('Batches') %></span>
|
21
21
|
<% end %>
|
22
|
+
|
23
|
+
<%= menu.nav_link(main_app.csv_edit_path) do %>
|
24
|
+
<span class="fa fa-angle-double-up"></span> <span class="sidebar-action-text"><%= t('Batch Update') %></span>
|
25
|
+
<% end %>
|
22
26
|
<% end %>
|
23
27
|
</li>
|
24
28
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cdm_migrator
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- sephirothkod
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-08-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rails
|
@@ -81,6 +81,7 @@ files:
|
|
81
81
|
- app/assets/config/cdm_migrator_manifest.js
|
82
82
|
- app/assets/javascripts/cdm_migrator/application.js
|
83
83
|
- app/assets/stylesheets/cdm_migrator/application.css
|
84
|
+
- app/assets/stylesheets/cdm_migrator/csv_checker.css
|
84
85
|
- app/controllers/cdm_migrator/application_controller.rb
|
85
86
|
- app/controllers/cdm_migrator/cdm_controller.rb
|
86
87
|
- app/controllers/cdm_migrator/csv_controller.rb
|
@@ -90,6 +91,7 @@ files:
|
|
90
91
|
- app/jobs/cdm_migrator/batch_create_works_job.rb
|
91
92
|
- app/jobs/cdm_migrator/cdm_ingest_files_job.rb
|
92
93
|
- app/jobs/cdm_migrator/create_work_job.rb
|
94
|
+
- app/jobs/cdm_migrator/update_object_job.rb
|
93
95
|
- app/mailers/cdm_migrator/application_mailer.rb
|
94
96
|
- app/models/cdm_migrator/application_record.rb
|
95
97
|
- app/models/cdm_migrator/batch_ingest.rb
|
@@ -102,7 +104,8 @@ files:
|
|
102
104
|
- app/views/cdm_migrator/csv/_path_list.html.erb
|
103
105
|
- app/views/cdm_migrator/csv/_results_pagination.html.erb
|
104
106
|
- app/views/cdm_migrator/csv/_tabs.html.erb
|
105
|
-
- app/views/cdm_migrator/csv/
|
107
|
+
- app/views/cdm_migrator/csv/csv_checker.html.erb
|
108
|
+
- app/views/cdm_migrator/csv/edit.html.erb
|
106
109
|
- app/views/cdm_migrator/csv/index.html.erb
|
107
110
|
- app/views/cdm_migrator/csv/upload.html.erb
|
108
111
|
- app/views/layouts/cdm_migrator/application.html.erb
|
@@ -1,21 +0,0 @@
|
|
1
|
-
<% provide :page_title, "File Path Checker" %>
|
2
|
-
|
3
|
-
<h1>File Path Checker</h1>
|
4
|
-
|
5
|
-
<!-- <div class="row"> -->
|
6
|
-
<p>This tool checks if a file exists at each url in a csv.</p>
|
7
|
-
<%= form_tag(check_file_paths_path, remote: true, method: :post, multipart: true, id: "csv-form") do %>
|
8
|
-
<div class="input-group">
|
9
|
-
<%= file_field_tag(:file, class: "form-control-file") %>
|
10
|
-
<%= hidden_field_tag :authenticity_token, value: form_authenticity_token %>
|
11
|
-
<%= button_tag(type: :submit, class: "btn btn-large btn-primary", style: "margin-top: 1em;") do %>
|
12
|
-
Check CSV
|
13
|
-
<% end %>
|
14
|
-
<% end %>
|
15
|
-
</div>
|
16
|
-
<!-- </div> -->
|
17
|
-
|
18
|
-
<div id="path_list" class="col-md-10 offset-md-1">
|
19
|
-
<%= render 'path_list' %>
|
20
|
-
</div>
|
21
|
-
|