cdm_migrator 3.4.2 → 4.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/MIT-LICENSE +0 -0
- data/README.md +0 -0
- data/Rakefile +0 -0
- data/app/assets/config/cdm_migrator_manifest.js +0 -0
- data/app/assets/javascripts/cdm_migrator/application.js +0 -0
- data/app/assets/stylesheets/cdm_migrator/application.css +0 -0
- data/app/assets/stylesheets/cdm_migrator/csv_checker.css +0 -0
- data/app/controllers/cdm_migrator/cdm_controller.rb +0 -0
- data/app/controllers/cdm_migrator/csv_controller.rb +279 -381
- data/app/helpers/cdm_migrator/application_helper.rb +0 -0
- data/app/jobs/cdm_migrator/application_job.rb +0 -0
- data/app/jobs/cdm_migrator/batch_create_files_job.rb +0 -0
- data/app/jobs/cdm_migrator/batch_create_files_with_ordered_members_job.rb +49 -22
- data/app/jobs/cdm_migrator/batch_create_works_job.rb +0 -0
- data/app/jobs/cdm_migrator/cdm_ingest_files_job.rb +0 -0
- data/app/jobs/cdm_migrator/create_work_job.rb +1 -1
- data/app/jobs/cdm_migrator/restart_upload_from_middle_job.rb +0 -0
- data/app/jobs/cdm_migrator/update_object_job.rb +0 -0
- data/app/mailers/cdm_migrator/application_mailer.rb +0 -0
- data/app/models/cdm_migrator/application_record.rb +0 -0
- data/app/models/cdm_migrator/batch_ingest.rb +33 -33
- data/app/models/cdm_migrator/csv_checker.rb +195 -0
- data/app/models/cdm_migrator/ingest_work.rb +7 -7
- data/app/services/{csv_export_service.rb → cdm_migrator/csv_export_service.rb} +0 -0
- data/app/views/cdm_migrator/cdm/collection.html.erb +0 -0
- data/app/views/cdm_migrator/cdm/mappings.html.erb +0 -0
- data/app/views/cdm_migrator/csv/_batches_list.html.erb +0 -0
- data/app/views/cdm_migrator/csv/_default_group.html.erb +0 -0
- data/app/views/cdm_migrator/csv/_error_list.html.erb +0 -0
- data/app/views/cdm_migrator/csv/_list_batches.html.erb +0 -0
- data/app/views/cdm_migrator/csv/_tabs.html.erb +0 -0
- data/app/views/cdm_migrator/csv/csv_checker.html.erb +39 -46
- data/app/views/cdm_migrator/csv/edit.html.erb +36 -17
- data/app/views/cdm_migrator/csv/index.html.erb +0 -0
- data/app/views/cdm_migrator/csv/upload.html.erb +52 -18
- data/app/views/layouts/cdm_migrator/application.html.erb +0 -0
- data/config/routes.rb +0 -0
- data/db/migrate/20191211193859_create_batch_ingests.rb +0 -0
- data/db/migrate/20191212192315_create_ingest_works.rb +20 -18
- data/lib/cdm_migrator/engine.rb +0 -0
- data/lib/cdm_migrator/version.rb +1 -1
- data/lib/cdm_migrator.rb +0 -0
- data/lib/generators/cdm_migrator/install/install_generator.rb +0 -0
- data/lib/generators/cdm_migrator/install/templates/config/cdm_migrator.yml +0 -0
- data/lib/generators/cdm_migrator/install/templates/presenters/hyku/menu_presenter.rb +0 -0
- data/lib/generators/cdm_migrator/install/templates/presenters/hyrax/menu_presenter.rb +0 -0
- data/lib/generators/cdm_migrator/install/templates/sidebar/_tasks.html.erb +0 -0
- data/lib/tasks/cdm_migrator_tasks.rake +0 -0
- metadata +16 -16
- data/app/controllers/cdm_migrator/application_controller.rb +0 -10
|
@@ -1,382 +1,280 @@
|
|
|
1
|
-
module CdmMigrator
|
|
2
|
-
class CsvController < ApplicationController
|
|
3
|
-
|
|
4
|
-
include ActionView::Helpers::UrlHelper
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
obj
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
def
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
@
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
end
|
|
217
|
-
|
|
218
|
-
def
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
@error_list[row_number].merge!(uri_errors)
|
|
281
|
-
else
|
|
282
|
-
@error_list[row_number] = uri_errors
|
|
283
|
-
end
|
|
284
|
-
end
|
|
285
|
-
|
|
286
|
-
# Check multi-value separators
|
|
287
|
-
def check_multi_val_fields(row_number, row, character)
|
|
288
|
-
uri_fields = @separator_fields
|
|
289
|
-
separator_errors = uri_fields.each_with_object({}) do |field, hash|
|
|
290
|
-
if value = row[field]
|
|
291
|
-
# Check for leading or trailing spaces
|
|
292
|
-
if value.match %r{ #{Regexp.escape(character)}|#{Regexp.escape(character)} }
|
|
293
|
-
hash[field.to_s] = "Contains leading or trailing whitespace around multi-value separator."
|
|
294
|
-
end
|
|
295
|
-
values = value.split(character).map(&:strip)
|
|
296
|
-
values.each do |val|
|
|
297
|
-
if val.match(URI.regexp) # Val should be URI
|
|
298
|
-
remainder = val.gsub(val.match(URI.regexp)[0],'')
|
|
299
|
-
unless remainder.blank?
|
|
300
|
-
hash[field.to_s] = "May contain the wrong multi-value separator or a typo in the URI."
|
|
301
|
-
end
|
|
302
|
-
else # Or val should be string
|
|
303
|
-
invalid_chars = ["\\"]
|
|
304
|
-
# Make exceptions for backslashes that are part of whitespace characters
|
|
305
|
-
# by deleting them before checking for stray \s
|
|
306
|
-
if val.delete("\t\r\n\s\n").match Regexp.union(invalid_chars)
|
|
307
|
-
hash[field.to_s] = "May contain an invalid character such as #{invalid_chars.to_sentence(last_word_connector: ", or ")}."
|
|
308
|
-
end
|
|
309
|
-
end
|
|
310
|
-
end
|
|
311
|
-
end
|
|
312
|
-
end
|
|
313
|
-
@error_list[row_number].merge!(separator_errors)
|
|
314
|
-
end
|
|
315
|
-
|
|
316
|
-
def default_page_title
|
|
317
|
-
'CSV Batch Uploader'
|
|
318
|
-
end
|
|
319
|
-
|
|
320
|
-
def admin_host?
|
|
321
|
-
false unless Settings.multitenancy.enabled
|
|
322
|
-
end
|
|
323
|
-
|
|
324
|
-
def available_translations
|
|
325
|
-
{
|
|
326
|
-
'en' => 'English',
|
|
327
|
-
'fr' => 'French'
|
|
328
|
-
}
|
|
329
|
-
end
|
|
330
|
-
|
|
331
|
-
def work_form(worktype = "GenericWork")
|
|
332
|
-
Module.const_get("Hyrax::#{worktype}Form") rescue nil || Module.const_get("Hyrax::Forms::WorkForm")
|
|
333
|
-
end
|
|
334
|
-
|
|
335
|
-
def file_form
|
|
336
|
-
Module.const_get("Hyrax::FileSetForm") rescue nil || Module.const_get("Hyrax::Forms::FileSetEditForm")
|
|
337
|
-
end
|
|
338
|
-
|
|
339
|
-
def secondary_terms form_name
|
|
340
|
-
form_name.terms - form_name.required_fields -
|
|
341
|
-
[:visibility_during_embargo, :embargo_release_date,
|
|
342
|
-
:visibility_after_embargo, :visibility_during_lease,
|
|
343
|
-
:lease_expiration_date, :visibility_after_lease, :visibility,
|
|
344
|
-
:thumbnail_id, :representative_id, :ordered_member_ids,
|
|
345
|
-
:collection_ids, :in_works_ids, :admin_set_id, :files, :source, :member_of_collection_ids]
|
|
346
|
-
end
|
|
347
|
-
|
|
348
|
-
def create_data data, type, object, mvs
|
|
349
|
-
final_data = {}
|
|
350
|
-
accepted_terms = type.required_fields + secondary_terms(type)
|
|
351
|
-
data.each do |key, att|
|
|
352
|
-
if (att.nil? || att.empty? || key.to_s.include?("object_type") || !accepted_terms.include?(key.to_sym))
|
|
353
|
-
next
|
|
354
|
-
elsif object.send(key).nil?
|
|
355
|
-
final_data[key] = att
|
|
356
|
-
else
|
|
357
|
-
final_data[key] = att.split(mvs)
|
|
358
|
-
end
|
|
359
|
-
end
|
|
360
|
-
final_data
|
|
361
|
-
end
|
|
362
|
-
|
|
363
|
-
def create_lease visibility, status_after, date
|
|
364
|
-
lease = Hydra::AccessControls::Lease.new(visibility_during_lease: visibility,
|
|
365
|
-
visibility_after_lease: status_after, lease_expiration_date: @lease_date)
|
|
366
|
-
lease.save
|
|
367
|
-
end
|
|
368
|
-
|
|
369
|
-
def create_embargo visibility
|
|
370
|
-
embargo = Hydra::AccessControls::Embargo.new
|
|
371
|
-
embargo.visibility_during_embargo = visibility
|
|
372
|
-
embargo.visibility_after_embargo = @status_after
|
|
373
|
-
embargo.embargo_release_date = @embargo_date
|
|
374
|
-
embargo.save
|
|
375
|
-
end
|
|
376
|
-
|
|
377
|
-
def log(user)
|
|
378
|
-
Hyrax::Operation.create!(user: user,
|
|
379
|
-
operation_type: "Attach Remote File")
|
|
380
|
-
end
|
|
381
|
-
end
|
|
1
|
+
module CdmMigrator
|
|
2
|
+
class CsvController < ::ApplicationController
|
|
3
|
+
# Needed for #current_page?
|
|
4
|
+
include ActionView::Helpers::UrlHelper
|
|
5
|
+
|
|
6
|
+
layout 'hyrax/dashboard' if Hyrax
|
|
7
|
+
|
|
8
|
+
before_action :authenticate, except: :index
|
|
9
|
+
before_action :load_config, only: :csv_checker
|
|
10
|
+
|
|
11
|
+
def csv_checker
|
|
12
|
+
if params[:file]
|
|
13
|
+
results = CsvChecker.new.check_csv(params[:file].path)
|
|
14
|
+
# Set warning messages
|
|
15
|
+
results['alerts'].each { |alert| flash[:alert] = alert }
|
|
16
|
+
if results['errors'].nil?
|
|
17
|
+
redirect_to csv_checker_path, notice: "All data are valid."
|
|
18
|
+
else
|
|
19
|
+
flash[:error] = "The CSV Checker found some errors in the CSV. Please correct them and check again."
|
|
20
|
+
@error_list = results['errors']
|
|
21
|
+
render :csv_checker
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def index
|
|
27
|
+
if current_page?(main_app.csv_my_batches_path(locale: nil))
|
|
28
|
+
@batches = BatchIngest.where(user_id: current_user.id).reverse_order
|
|
29
|
+
elsif current_page?(main_app.csv_all_batches_path(locale: nil))
|
|
30
|
+
@batches = BatchIngest.all.reverse_order
|
|
31
|
+
else
|
|
32
|
+
@batches = []
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def upload
|
|
37
|
+
@admin_sets = AdminSet.all.map { |as| [as.title.first, as.id] }
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def create
|
|
41
|
+
csv_upload_params
|
|
42
|
+
dir = Rails.root.join('public', 'uploads', 'csvs')
|
|
43
|
+
FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
|
|
44
|
+
time = DateTime.now.strftime('%s')
|
|
45
|
+
filename = params[:csv_import][:csv_file].original_filename.gsub('.csv', "#{time}.csv")
|
|
46
|
+
csv = dir.join(filename).to_s
|
|
47
|
+
File.open(csv, 'wb') do |file|
|
|
48
|
+
file.write(params[:csv_import][:csv_file].read)
|
|
49
|
+
end
|
|
50
|
+
check_csv csv
|
|
51
|
+
if @error_list.present?
|
|
52
|
+
flash[:error] = "Cdm Migrator found some problems with the CSV. Use the CSV Checker for more details."
|
|
53
|
+
end
|
|
54
|
+
parse_csv(csv, params[:csv_import][:mvs])
|
|
55
|
+
|
|
56
|
+
ingest = BatchIngest.new({
|
|
57
|
+
data: @works,
|
|
58
|
+
size: @works.length,
|
|
59
|
+
csv: csv,
|
|
60
|
+
admin_set_id: params[:csv_import][:admin_set],
|
|
61
|
+
collection_id: params[:csv_import][:collection],
|
|
62
|
+
user_id: current_user.id,
|
|
63
|
+
message: @path_list.blank? ? nil : @path_list.to_s.gsub("\"", """)
|
|
64
|
+
})
|
|
65
|
+
if ingest.save! && @path_list.blank?
|
|
66
|
+
BatchCreateWorksJob.perform_later(ingest, current_user)
|
|
67
|
+
flash[:notice] = "csv successfully uploaded, check this page to see the status while the batch is running"
|
|
68
|
+
redirect_to csv_my_batches_path
|
|
69
|
+
else
|
|
70
|
+
flash[:error] ||= "csv could not be parsed, please check and re-upload"
|
|
71
|
+
redirect_to csv_upload_path
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def rerun
|
|
76
|
+
ingest = BatchIngest.find(params[:id]).deep_dup
|
|
77
|
+
ingest.save
|
|
78
|
+
BatchCreateWorksJob.perform_later(ingest, current_user)
|
|
79
|
+
flash[:notice] = "csv successfully uploaded, check this page to see the status while the batch is running"
|
|
80
|
+
redirect_to csv_my_batches_path
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def generate
|
|
84
|
+
headers = %w(type url)
|
|
85
|
+
skip = %w(id head tail depositor date_uploaded date_modified import_url thumbnail_id embargo_id lease_id access_control_id representative_id)
|
|
86
|
+
GenericWork.new.attributes.each do |key, val|
|
|
87
|
+
headers << "work_#{key}" unless skip.include? key
|
|
88
|
+
end
|
|
89
|
+
FileSet.new.attributes.each do |key, val|
|
|
90
|
+
headers << "file_#{key}" unless skip.include? key
|
|
91
|
+
end
|
|
92
|
+
fname = "template_#{DateTime.now.to_i}"
|
|
93
|
+
render plain: CSV.generate { |csv| csv << headers }, content_type: 'text/csv'
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def edit
|
|
97
|
+
# Intentionally blank
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def update
|
|
101
|
+
mvs = params[:csv_update][:mvs]
|
|
102
|
+
csv = CSV.parse(params[:csv_update][:csv_file].read.force_encoding("UTF-8"), headers: true, encoding: 'utf-8').map(&:to_hash)
|
|
103
|
+
csv.each do |row|
|
|
104
|
+
obj = ActiveFedora::Base.find row['id']
|
|
105
|
+
type = row.first.last
|
|
106
|
+
if type.nil?
|
|
107
|
+
next
|
|
108
|
+
elsif type.include? "Work"
|
|
109
|
+
metadata = create_data(row.except('id', 'type'), work_form(type), obj, mvs)
|
|
110
|
+
elsif type.include? "File"
|
|
111
|
+
metadata = create_data(row.except('id', 'type'), file_form, obj, mvs)
|
|
112
|
+
end
|
|
113
|
+
unless metadata.nil?
|
|
114
|
+
obj.attributes = metadata
|
|
115
|
+
obj.try(:to_controlled_vocab)
|
|
116
|
+
obj.save
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
flash[:notice] = "csv successfully uploaded"
|
|
120
|
+
redirect_to csv_edit_path
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def export
|
|
124
|
+
csv_export_params
|
|
125
|
+
# Get a collection's member works from Solr
|
|
126
|
+
solr = RSolr.connect url: Blacklight.connection_config[:url]
|
|
127
|
+
response = solr.get 'select', params: {
|
|
128
|
+
q: "member_of_collection_ids_ssim:#{params[:csv_export][:collection]}",
|
|
129
|
+
fq: ["has_model_ssim:FileSet OR has_model_ssim:*Work"],
|
|
130
|
+
rows: 3400,
|
|
131
|
+
fl: "id"
|
|
132
|
+
}
|
|
133
|
+
unless response['response']['docs'].empty? || response['response']['docs'][0].empty?
|
|
134
|
+
work_ids = response['response']['docs'].map { |doc| doc['id'] }
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
send_data CsvExportService.new(available_works).csv_for(work_ids),
|
|
138
|
+
:type => 'text/csv; charset=iso-8859-5; header=present',
|
|
139
|
+
:disposition => "attachment; filename=export.csv"
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
private
|
|
143
|
+
|
|
144
|
+
def authenticate
|
|
145
|
+
authorize! :create, available_works.first
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def csv_upload_params
|
|
149
|
+
params.require(:csv_import).permit(:csv_file, :mvs, :admin_set, :collection)
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def csv_export_params
|
|
153
|
+
params.require(:csv_export).permit(:collection)
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def available_works
|
|
157
|
+
@available_works ||= Hyrax::QuickClassificationQuery.new(current_user).authorized_models
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
def parse_csv csv, mvs
|
|
161
|
+
csv = CSV.parse(File.read(csv), headers: true, encoding: 'utf-8').map(&:to_hash)
|
|
162
|
+
@works = []
|
|
163
|
+
csv.each do |row|
|
|
164
|
+
type = row.first.last
|
|
165
|
+
if type.nil?
|
|
166
|
+
next
|
|
167
|
+
elsif type.include? "Work"
|
|
168
|
+
metadata = create_data(row, work_form(type), Object.const_get(type).new, mvs)
|
|
169
|
+
@works << {type: type, metadata: metadata, files: []}
|
|
170
|
+
elsif type.include? "File"
|
|
171
|
+
metadata = create_data(row, file_form, FileSet.new, mvs)
|
|
172
|
+
@works.last[:files] << {url: row.delete('url'), title: row.delete('title'), metadata: metadata}
|
|
173
|
+
end
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def load_config
|
|
178
|
+
# multitenant? defined in ApplicationController
|
|
179
|
+
if multitenant?
|
|
180
|
+
tenant = Account.find_by(tenant: Apartment::Tenant.current).cname
|
|
181
|
+
else
|
|
182
|
+
tenant = "default"
|
|
183
|
+
end
|
|
184
|
+
if CdmMigrator::Engine.config['tenant_settings'].has_key?(tenant)
|
|
185
|
+
settings = CdmMigrator::Engine.config['tenant_settings'][tenant]['csv_checker']
|
|
186
|
+
if settings.present?
|
|
187
|
+
@date_indexing_service = settings['date_indexing_service'].first.constantize if settings['date_indexing_service']
|
|
188
|
+
@date_fields = settings['date_fields'].map(&:to_sym) if settings['date_fields']
|
|
189
|
+
@uri_fields = settings['valid_uri_fields'].map(&:to_sym) if settings['valid_uri_fields']
|
|
190
|
+
@separator = settings['multi_value_separator']
|
|
191
|
+
@separator_fields = settings['separator_fields'].map(&:to_sym) if settings['separator_fields']
|
|
192
|
+
@path_to_drive = settings['path_to_drive']
|
|
193
|
+
# If you would like to change this to match the uploader's max file size,
|
|
194
|
+
# change this to Hyrax.config.uploader[:maxFileSize]
|
|
195
|
+
@max_file_size = settings['max_file_size']
|
|
196
|
+
else
|
|
197
|
+
raise "Cdm Migrator couldn't find any configured settings. Are they in cdm_migrator.yml?"
|
|
198
|
+
end
|
|
199
|
+
else
|
|
200
|
+
raise "Cdm Migrator couldn't find this tenant. Is it configured?"
|
|
201
|
+
end
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
def check_csv csv_file
|
|
205
|
+
results = CsvChecker.new.check_csv(csv_file)
|
|
206
|
+
@error_list = results['errors']
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def default_page_title
|
|
210
|
+
'CSV Batch Uploader'
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
def admin_host?
|
|
214
|
+
# multitenant? defined in ApplicationController
|
|
215
|
+
false unless multitenant?
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
def available_translations
|
|
219
|
+
{
|
|
220
|
+
'en' => 'English',
|
|
221
|
+
'fr' => 'French'
|
|
222
|
+
}
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
def work_form(worktype = "GenericWork")
|
|
226
|
+
Module.const_get("Hyrax::#{worktype}Form") rescue nil || Module.const_get("Hyrax::Forms::WorkForm")
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
def file_form
|
|
230
|
+
Module.const_get("Hyrax::FileSetForm") rescue nil || Module.const_get("Hyrax::Forms::FileSetEditForm")
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
def secondary_terms form_name
|
|
234
|
+
form_name.terms - form_name.required_fields -
|
|
235
|
+
[:visibility_during_embargo, :embargo_release_date,
|
|
236
|
+
:visibility_after_embargo, :visibility_during_lease,
|
|
237
|
+
:lease_expiration_date, :visibility_after_lease, :visibility,
|
|
238
|
+
:thumbnail_id, :representative_id, :ordered_member_ids,
|
|
239
|
+
:collection_ids, :in_works_ids, :admin_set_id, :files, :source, :member_of_collection_ids]
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
def create_data data, type, object, mvs
|
|
243
|
+
final_data = {}
|
|
244
|
+
accepted_terms = type.required_fields + secondary_terms(type)
|
|
245
|
+
data.each do |key, att|
|
|
246
|
+
if (att.nil? || att.empty? || key.to_s.include?("object_type") || !accepted_terms.include?(key.to_sym))
|
|
247
|
+
next
|
|
248
|
+
elsif object.send(key).nil?
|
|
249
|
+
final_data[key] = att
|
|
250
|
+
else
|
|
251
|
+
if object.class.properties[key.to_s].multiple?
|
|
252
|
+
final_data[key] = att.split(mvs)
|
|
253
|
+
else
|
|
254
|
+
final_data[key] = att
|
|
255
|
+
end
|
|
256
|
+
end
|
|
257
|
+
end
|
|
258
|
+
final_data
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
def create_lease visibility, status_after, date
|
|
262
|
+
lease = Hydra::AccessControls::Lease.new(visibility_during_lease: visibility,
|
|
263
|
+
visibility_after_lease: status_after, lease_expiration_date: @lease_date)
|
|
264
|
+
lease.save
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
def create_embargo visibility
|
|
268
|
+
embargo = Hydra::AccessControls::Embargo.new
|
|
269
|
+
embargo.visibility_during_embargo = visibility
|
|
270
|
+
embargo.visibility_after_embargo = @status_after
|
|
271
|
+
embargo.embargo_release_date = @embargo_date
|
|
272
|
+
embargo.save
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
def log(user)
|
|
276
|
+
Hyrax::Operation.create!(user: user,
|
|
277
|
+
operation_type: "Attach Remote File")
|
|
278
|
+
end
|
|
279
|
+
end
|
|
382
280
|
end
|