cdm_migrator 3.5.2 → 4.6.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/MIT-LICENSE +0 -0
  3. data/README.md +0 -0
  4. data/Rakefile +0 -0
  5. data/app/assets/config/cdm_migrator_manifest.js +0 -0
  6. data/app/assets/javascripts/cdm_migrator/application.js +0 -0
  7. data/app/assets/stylesheets/cdm_migrator/application.css +0 -0
  8. data/app/assets/stylesheets/cdm_migrator/csv_checker.css +0 -0
  9. data/app/controllers/cdm_migrator/cdm_controller.rb +0 -0
  10. data/app/controllers/cdm_migrator/csv_controller.rb +279 -381
  11. data/app/helpers/cdm_migrator/application_helper.rb +0 -0
  12. data/app/jobs/cdm_migrator/application_job.rb +0 -0
  13. data/app/jobs/cdm_migrator/batch_create_files_job.rb +0 -0
  14. data/app/jobs/cdm_migrator/batch_create_files_with_ordered_members_job.rb +0 -0
  15. data/app/jobs/cdm_migrator/batch_create_works_job.rb +0 -0
  16. data/app/jobs/cdm_migrator/cdm_ingest_files_job.rb +0 -0
  17. data/app/jobs/cdm_migrator/create_work_job.rb +1 -1
  18. data/app/jobs/cdm_migrator/restart_upload_from_middle_job.rb +0 -0
  19. data/app/jobs/cdm_migrator/update_object_job.rb +0 -0
  20. data/app/mailers/cdm_migrator/application_mailer.rb +0 -0
  21. data/app/models/cdm_migrator/application_record.rb +0 -0
  22. data/app/models/cdm_migrator/batch_ingest.rb +33 -33
  23. data/app/models/cdm_migrator/csv_checker.rb +195 -0
  24. data/app/models/cdm_migrator/ingest_work.rb +7 -7
  25. data/app/services/{csv_export_service.rb → cdm_migrator/csv_export_service.rb} +0 -0
  26. data/app/views/cdm_migrator/cdm/collection.html.erb +0 -0
  27. data/app/views/cdm_migrator/cdm/mappings.html.erb +0 -0
  28. data/app/views/cdm_migrator/csv/_batches_list.html.erb +0 -0
  29. data/app/views/cdm_migrator/csv/_default_group.html.erb +0 -0
  30. data/app/views/cdm_migrator/csv/_error_list.html.erb +0 -0
  31. data/app/views/cdm_migrator/csv/_list_batches.html.erb +0 -0
  32. data/app/views/cdm_migrator/csv/_tabs.html.erb +0 -0
  33. data/app/views/cdm_migrator/csv/csv_checker.html.erb +39 -46
  34. data/app/views/cdm_migrator/csv/edit.html.erb +36 -17
  35. data/app/views/cdm_migrator/csv/index.html.erb +0 -0
  36. data/app/views/cdm_migrator/csv/upload.html.erb +52 -18
  37. data/app/views/layouts/cdm_migrator/application.html.erb +0 -0
  38. data/config/routes.rb +0 -0
  39. data/db/migrate/20191211193859_create_batch_ingests.rb +0 -0
  40. data/db/migrate/20191212192315_create_ingest_works.rb +20 -18
  41. data/lib/cdm_migrator/engine.rb +0 -0
  42. data/lib/cdm_migrator/version.rb +1 -1
  43. data/lib/cdm_migrator.rb +0 -0
  44. data/lib/generators/cdm_migrator/install/install_generator.rb +0 -0
  45. data/lib/generators/cdm_migrator/install/templates/config/cdm_migrator.yml +0 -0
  46. data/lib/generators/cdm_migrator/install/templates/presenters/hyku/menu_presenter.rb +0 -0
  47. data/lib/generators/cdm_migrator/install/templates/presenters/hyrax/menu_presenter.rb +0 -0
  48. data/lib/generators/cdm_migrator/install/templates/sidebar/_tasks.html.erb +0 -0
  49. data/lib/tasks/cdm_migrator_tasks.rake +0 -0
  50. metadata +16 -16
  51. data/app/controllers/cdm_migrator/application_controller.rb +0 -10
@@ -1,382 +1,280 @@
1
- module CdmMigrator
2
- class CsvController < ApplicationController
3
- helper_method :default_page_title, :admin_host?, :available_translations, :available_works
4
- include ActionView::Helpers::UrlHelper
5
- layout 'hyrax/dashboard' if Hyrax
6
- before_action :authenticate, except: :index
7
- before_action :load_config, only: :csv_checker
8
-
9
- def csv_checker
10
- if params[:file]
11
- check_csv params[:file].path
12
- if @error_list.blank?
13
- flash[:notice] = "All data are valid."
14
- else
15
- flash[:error] = "The CSV Checker found some errors in the CSV. Please correct them and check again."
16
- end
17
- end
18
- end
19
-
20
- def index
21
- if current_page?(main_app.csv_my_batches_path(locale: nil))
22
- @batches = BatchIngest.where(user_id: current_user.id).reverse_order
23
- elsif current_page?(main_app.csv_all_batches_path(locale: nil))
24
- @batches = BatchIngest.all.reverse_order
25
- else
26
- @batches = []
27
- end
28
- end
29
-
30
- def upload
31
- @admin_sets = AdminSet.all.map { |as| [as.title.first, as.id] }
32
- @collections = Hyrax.config.collection_class.all.map { |col| [col.title.first, col.id] }
33
- end
34
-
35
- def create
36
- dir = Rails.root.join('public', 'uploads', 'csvs')
37
- FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
38
- time = DateTime.now.strftime('%s')
39
- filename = params[:csv_import][:csv_file].original_filename.gsub('.csv', "#{time}.csv")
40
- csv = dir.join(filename).to_s
41
- File.open(csv, 'wb') do |file|
42
- file.write(params[:csv_import][:csv_file].read)
43
- end
44
- check_csv csv
45
- if @error_list.present?
46
- flash[:error] = "Cdm Migrator found some problems with the CSV. Use the CSV Checker for more details."
47
- end
48
- parse_csv(csv, params[:csv_import][:mvs])
49
-
50
- ingest = BatchIngest.new({
51
- data: @works,
52
- size: @works.length,
53
- csv: csv,
54
- admin_set_id: params[:admin_set],
55
- collection_id: params[:collection],
56
- user_id: current_user.id,
57
- message: @path_list.blank? ? nil : @path_list.to_s.gsub("\"", "&quot;")
58
- })
59
- if ingest.save! && @path_list.blank?
60
- BatchCreateWorksJob.perform_later(ingest, current_user)
61
- flash[:notice] = "csv successfully uploaded, check this page to see the status while the batch is running"
62
- redirect_to csv_my_batches_path
63
- else
64
- flash[:error] ||= "csv could not be parsed, please check and re-upload"
65
- redirect_to csv_upload_path
66
- end
67
- end
68
-
69
- def rerun
70
- ingest = BatchIngest.find(params[:id]).deep_dup
71
- ingest.save
72
- BatchCreateWorksJob.perform_later(ingest, current_user)
73
- flash[:notice] = "csv successfully uploaded, check this page to see the status while the batch is running"
74
- redirect_to csv_my_batches_path
75
- end
76
-
77
- def generate
78
- headers = %w(type url)
79
- skip = %w(id head tail depositor date_uploaded date_modified import_url thumbnail_id embargo_id lease_id access_control_id representative_id)
80
- GenericWork.new.attributes.each do |key, val|
81
- headers << "work_#{key}" unless skip.include? key
82
- end
83
- FileSet.new.attributes.each do |key, val|
84
- headers << "file_#{key}" unless skip.include? key
85
- end
86
- fname = "template_#{DateTime.now.to_i}"
87
- render plain: CSV.generate { |csv| csv << headers }, content_type: 'text/csv'
88
- end
89
-
90
- def edit
91
- @collections = Hyrax.config.collection_class.all.map { |c| [c.title.first, c.id] }
92
- end
93
-
94
- def update
95
- mvs = params[:csv_update][:mvs]
96
- csv = CSV.parse(params[:csv_update][:csv_file].read.force_encoding("UTF-8"), headers: true, encoding: 'utf-8').map(&:to_hash)
97
- csv.each do |row|
98
- obj = ActiveFedora::Base.find row['id']
99
- type = row.first.last
100
- if type.nil?
101
- next
102
- elsif type.include? "Work"
103
- metadata = create_data(row.except('id', 'type'), work_form(type), obj, mvs)
104
- elsif type.include? "File"
105
- metadata = create_data(row.except('id', 'type'), file_form, obj, mvs)
106
- end
107
- unless metadata.nil?
108
- obj.attributes = metadata
109
- obj.save
110
- end
111
- end
112
- flash[:notice] = "csv successfully uploaded"
113
- redirect_to csv_edit_path
114
- end
115
-
116
- def export
117
- # Get a collection's member works from Solr
118
- solr = RSolr.connect url: Blacklight.connection_config[:url]
119
- response = solr.get 'select', params: {
120
- q: "member_of_collection_ids_ssim:#{params[:collection_id]}",
121
- fq: ["has_model_ssim:FileSet OR has_model_ssim:*Work"],
122
- rows: 3400,
123
- fl: "id"
124
- }
125
- unless response['response']['docs'].empty? || response['response']['docs'][0].empty?
126
- work_ids = response['response']['docs'].map { |doc| doc['id'] }
127
- end
128
-
129
- send_data CsvExportService.new(available_works).csv_for(work_ids),
130
- :type => 'text/csv; charset=iso-8859-5; header=present',
131
- :disposition => "attachment; filename=export.csv"
132
- end
133
-
134
- private
135
-
136
- def authenticate
137
- authorize! :create, available_works.first
138
- end
139
-
140
- def available_works
141
- @available_works ||= Hyrax::QuickClassificationQuery.new(current_user).authorized_models
142
- end
143
-
144
- def parse_csv csv, mvs
145
- csv = CSV.parse(File.read(csv), headers: true, encoding: 'utf-8').map(&:to_hash)
146
- @works = []
147
- csv.each do |row|
148
- type = row.first.last
149
- if type.nil?
150
- next
151
- elsif type.include? "Work"
152
- metadata = create_data(row, work_form(type), Object.const_get(type).new, mvs)
153
- @works << {type: type, metadata: metadata, files: []}
154
- elsif type.include? "File"
155
- metadata = create_data(row, file_form, FileSet.new, mvs)
156
- @works.last[:files] << {url: row.delete('url'), title: row.delete('title'), metadata: metadata}
157
- end
158
- end
159
- end
160
-
161
- def load_config
162
- if Settings.multitenancy.enabled
163
- tenant = Account.find_by(tenant: Apartment::Tenant.current).cname
164
- else
165
- tenant = "default"
166
- end
167
- if CdmMigrator::Engine.config['tenant_settings'].has_key?(tenant)
168
- settings = CdmMigrator::Engine.config['tenant_settings'][tenant]['csv_checker']
169
- if settings.present?
170
- @date_indexing_service = settings['date_indexing_service'].first.constantize if settings['date_indexing_service']
171
- @date_fields = settings['date_fields'].map(&:to_sym) if settings['date_fields']
172
- @uri_fields = settings['valid_uri_fields'].map(&:to_sym) if settings['valid_uri_fields']
173
- @separator = settings['multi_value_separator']
174
- @separator_fields = settings['separator_fields'].map(&:to_sym) if settings['separator_fields']
175
- @path_to_drive = settings['path_to_drive']
176
- # If you would like to change this to match the uploader's max file size,
177
- # change this to Hyrax.config.uploader[:maxFileSize]
178
- @max_file_size = settings['max_file_size']
179
- else
180
- raise "Cdm Migrator couldn't find any configured settings. Are they in cdm_migrator.yml?"
181
- end
182
- else
183
- raise "Cdm Migrator couldn't find this tenant. Is it configured?"
184
- end
185
- end
186
-
187
- def check_csv csv_file
188
- row_number = 1
189
- @error_list = {}
190
- check_mounted_drive if @path_to_drive.present?
191
-
192
- CSV.foreach(csv_file, headers: true, header_converters: :symbol) do |row|
193
- row_number +=1 # Tells user what CSV row the error is on
194
- if row[:object_type].include? "Work"
195
- check_dates(row_number, row) if @date_fields.present?
196
- check_uris(row_number, row) if @uri_fields.present?
197
- if params[:multi_value_separator].present? and @separator_fields.present?
198
- check_multi_val_fields(row_number, row, params[:multi_value_separator])
199
- else
200
- alert_message = "No multi-value separator character was selected or no fields were configured. CSV Checker didn't check for valid separators."
201
- if flash[:alert] and flash[:alert].exclude?(alert_message) # Only add this message once, rather than per line
202
- flash[:alert] << alert_message
203
- elsif flash[:alert].blank?
204
- flash[:alert] = Array.wrap(alert_message)
205
- end
206
- end
207
- elsif row[:object_type] == "File"
208
- check_file_path(row_number, row[:url])
209
- check_transcript_length(row_number, row[:transcript]) if row[:transcript].present?
210
- check_file_size(row_number, row[:url])
211
- else
212
- @error_list[row_number] = { "object_type" => "No or unknown object type. Please give a valid type (e.g. GenericWork, File)." }
213
- end
214
- @error_list.delete_if { |key, value| value.blank? } # Data are valid, no need to print the row
215
- end
216
- end
217
-
218
- def check_transcript_length(row_number, transcript)
219
- if transcript.is_a? String
220
- if transcript.length > 9000
221
- @error_list[row_number] = { "transcript" => "Transcript is too long (over 9000 characters)." }
222
- end
223
- elsif transcript.is_a? Array
224
- if transcript.any? { |tr| tr.length > 9000 }
225
- @error_list[row_number] = { "transcript" => "Transcript is too long (over 9000 characters)." }
226
- end
227
- end
228
- end
229
-
230
- def check_file_size(row_number, file_path)
231
- if file_path.present? && File.file?(file_path) && @max_file_size
232
- if File.size(file_path.gsub("file://", "")) > @max_file_size
233
- @error_list[row_number] = { "file size" => "The file at #{file_path} is too large to be uploaded. Please compress the file or split it into parts.
234
- Each part should be under #{helpers.number_to_human_size(@max_file_size)}." }
235
- end
236
- end
237
- end
238
-
239
- def check_mounted_drive
240
- drive_address = @path_to_drive
241
- unless Dir.exist?(drive_address) and !Dir[drive_address].empty?
242
- flash[:alert] = "CSV Checker can't find the mounted drive to check file paths, so some paths may be mislabelled as incorrect. Please contact the administrator or try again later."
243
- end
244
- end
245
-
246
- def check_file_path(row_number, file_path)
247
- if file_path.nil?
248
- @error_list[row_number] = { "url" => "url is blank." }
249
- elsif File.file?(file_path.gsub("file://", "")) == false
250
- @error_list[row_number] = { "url" => "No file found at #{file_path}" }
251
- end
252
- end
253
-
254
- def check_dates(row_number, row)
255
- date_fields = @date_fields
256
- unless @date_indexing_service
257
- flash[:alert] = "No date indexing service was configured so CSV Checker didn't validate dates."
258
- return
259
- end
260
- edtf_errors = date_fields.each_with_object({}) do |field, hash|
261
- next unless row[field]
262
- begin
263
- @date_indexing_service.new(row[field])
264
- rescue *@date_indexing_service.error_classes => error
265
- hash[field.to_s] = "#{error.message}"
266
- end
267
- end
268
- @error_list[row_number] = edtf_errors
269
- end
270
-
271
- # <Example: should be http://rightsstatements.org/vocab/etc. NOT https://rightsstatements.org/page/etc.
272
- def check_uris(row_number, row)
273
- uri_fields = @uri_fields
274
- uri_errors = uri_fields.each_with_object({}) do |field, hash|
275
- if row[field] and row[field].include? "page"
276
- hash[field.to_s] = "Links to page instead of URI. (e.g. https://rightsstatements.org/page/etc. instead of http://rightsstatements.org/vocab/etc.)"
277
- end
278
- end
279
- if @error_list.any?
280
- @error_list[row_number].merge!(uri_errors)
281
- else
282
- @error_list[row_number] = uri_errors
283
- end
284
- end
285
-
286
- # Check multi-value separators
287
- def check_multi_val_fields(row_number, row, character)
288
- uri_fields = @separator_fields
289
- separator_errors = uri_fields.each_with_object({}) do |field, hash|
290
- if value = row[field]
291
- # Check for leading or trailing spaces
292
- if value.match %r{ #{Regexp.escape(character)}|#{Regexp.escape(character)} }
293
- hash[field.to_s] = "Contains leading or trailing whitespace around multi-value separator."
294
- end
295
- values = value.split(character).map(&:strip)
296
- values.each do |val|
297
- if val.match(URI.regexp) # Val should be URI
298
- remainder = val.gsub(val.match(URI.regexp)[0],'')
299
- unless remainder.blank?
300
- hash[field.to_s] = "May contain the wrong multi-value separator or a typo in the URI."
301
- end
302
- else # Or val should be string
303
- invalid_chars = ["\\"]
304
- # Make exceptions for backslashes that are part of whitespace characters
305
- # by deleting them before checking for stray \s
306
- if val.delete("\t\r\n\s\n").match Regexp.union(invalid_chars)
307
- hash[field.to_s] = "May contain an invalid character such as #{invalid_chars.to_sentence(last_word_connector: ", or ")}."
308
- end
309
- end
310
- end
311
- end
312
- end
313
- @error_list[row_number].merge!(separator_errors)
314
- end
315
-
316
- def default_page_title
317
- 'CSV Batch Uploader'
318
- end
319
-
320
- def admin_host?
321
- false unless Settings.multitenancy.enabled
322
- end
323
-
324
- def available_translations
325
- {
326
- 'en' => 'English',
327
- 'fr' => 'French'
328
- }
329
- end
330
-
331
- def work_form(worktype = "GenericWork")
332
- Module.const_get("Hyrax::#{worktype}Form") rescue nil || Module.const_get("Hyrax::Forms::WorkForm")
333
- end
334
-
335
- def file_form
336
- Module.const_get("Hyrax::FileSetForm") rescue nil || Module.const_get("Hyrax::Forms::FileSetEditForm")
337
- end
338
-
339
- def secondary_terms form_name
340
- form_name.terms - form_name.required_fields -
341
- [:visibility_during_embargo, :embargo_release_date,
342
- :visibility_after_embargo, :visibility_during_lease,
343
- :lease_expiration_date, :visibility_after_lease, :visibility,
344
- :thumbnail_id, :representative_id, :ordered_member_ids,
345
- :collection_ids, :in_works_ids, :admin_set_id, :files, :source, :member_of_collection_ids]
346
- end
347
-
348
- def create_data data, type, object, mvs
349
- final_data = {}
350
- accepted_terms = type.required_fields + secondary_terms(type)
351
- data.each do |key, att|
352
- if (att.nil? || att.empty? || key.to_s.include?("object_type") || !accepted_terms.include?(key.to_sym))
353
- next
354
- elsif object.send(key).nil?
355
- final_data[key] = att
356
- else
357
- final_data[key] = att.split(mvs)
358
- end
359
- end
360
- final_data
361
- end
362
-
363
- def create_lease visibility, status_after, date
364
- lease = Hydra::AccessControls::Lease.new(visibility_during_lease: visibility,
365
- visibility_after_lease: status_after, lease_expiration_date: @lease_date)
366
- lease.save
367
- end
368
-
369
- def create_embargo visibility
370
- embargo = Hydra::AccessControls::Embargo.new
371
- embargo.visibility_during_embargo = visibility
372
- embargo.visibility_after_embargo = @status_after
373
- embargo.embargo_release_date = @embargo_date
374
- embargo.save
375
- end
376
-
377
- def log(user)
378
- Hyrax::Operation.create!(user: user,
379
- operation_type: "Attach Remote File")
380
- end
381
- end
1
+ module CdmMigrator
2
+ class CsvController < ::ApplicationController
3
+ # Needed for #current_page?
4
+ include ActionView::Helpers::UrlHelper
5
+
6
+ layout 'hyrax/dashboard' if Hyrax
7
+
8
+ before_action :authenticate, except: :index
9
+ before_action :load_config, only: :csv_checker
10
+
11
+ def csv_checker
12
+ if params[:file]
13
+ results = CsvChecker.new.check_csv(params[:file].path)
14
+ # Set warning messages
15
+ results['alerts'].each { |alert| flash[:alert] = alert }
16
+ if results['errors'].nil?
17
+ redirect_to csv_checker_path, notice: "All data are valid."
18
+ else
19
+ flash[:error] = "The CSV Checker found some errors in the CSV. Please correct them and check again."
20
+ @error_list = results['errors']
21
+ render :csv_checker
22
+ end
23
+ end
24
+ end
25
+
26
+ def index
27
+ if current_page?(main_app.csv_my_batches_path(locale: nil))
28
+ @batches = BatchIngest.where(user_id: current_user.id).reverse_order
29
+ elsif current_page?(main_app.csv_all_batches_path(locale: nil))
30
+ @batches = BatchIngest.all.reverse_order
31
+ else
32
+ @batches = []
33
+ end
34
+ end
35
+
36
+ def upload
37
+ @admin_sets = AdminSet.all.map { |as| [as.title.first, as.id] }
38
+ end
39
+
40
+ def create
41
+ csv_upload_params
42
+ dir = Rails.root.join('public', 'uploads', 'csvs')
43
+ FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
44
+ time = DateTime.now.strftime('%s')
45
+ filename = params[:csv_import][:csv_file].original_filename.gsub('.csv', "#{time}.csv")
46
+ csv = dir.join(filename).to_s
47
+ File.open(csv, 'wb') do |file|
48
+ file.write(params[:csv_import][:csv_file].read)
49
+ end
50
+ check_csv csv
51
+ if @error_list.present?
52
+ flash[:error] = "Cdm Migrator found some problems with the CSV. Use the CSV Checker for more details."
53
+ end
54
+ parse_csv(csv, params[:csv_import][:mvs])
55
+
56
+ ingest = BatchIngest.new({
57
+ data: @works,
58
+ size: @works.length,
59
+ csv: csv,
60
+ admin_set_id: params[:csv_import][:admin_set],
61
+ collection_id: params[:csv_import][:collection],
62
+ user_id: current_user.id,
63
+ message: @path_list.blank? ? nil : @path_list.to_s.gsub("\"", "&quot;")
64
+ })
65
+ if ingest.save! && @path_list.blank?
66
+ BatchCreateWorksJob.perform_later(ingest, current_user)
67
+ flash[:notice] = "csv successfully uploaded, check this page to see the status while the batch is running"
68
+ redirect_to csv_my_batches_path
69
+ else
70
+ flash[:error] ||= "csv could not be parsed, please check and re-upload"
71
+ redirect_to csv_upload_path
72
+ end
73
+ end
74
+
75
+ def rerun
76
+ ingest = BatchIngest.find(params[:id]).deep_dup
77
+ ingest.save
78
+ BatchCreateWorksJob.perform_later(ingest, current_user)
79
+ flash[:notice] = "csv successfully uploaded, check this page to see the status while the batch is running"
80
+ redirect_to csv_my_batches_path
81
+ end
82
+
83
+ def generate
84
+ headers = %w(type url)
85
+ skip = %w(id head tail depositor date_uploaded date_modified import_url thumbnail_id embargo_id lease_id access_control_id representative_id)
86
+ GenericWork.new.attributes.each do |key, val|
87
+ headers << "work_#{key}" unless skip.include? key
88
+ end
89
+ FileSet.new.attributes.each do |key, val|
90
+ headers << "file_#{key}" unless skip.include? key
91
+ end
92
+ fname = "template_#{DateTime.now.to_i}"
93
+ render plain: CSV.generate { |csv| csv << headers }, content_type: 'text/csv'
94
+ end
95
+
96
+ def edit
97
+ # Intentionally blank
98
+ end
99
+
100
+ def update
101
+ mvs = params[:csv_update][:mvs]
102
+ csv = CSV.parse(params[:csv_update][:csv_file].read.force_encoding("UTF-8"), headers: true, encoding: 'utf-8').map(&:to_hash)
103
+ csv.each do |row|
104
+ obj = ActiveFedora::Base.find row['id']
105
+ type = row.first.last
106
+ if type.nil?
107
+ next
108
+ elsif type.include? "Work"
109
+ metadata = create_data(row.except('id', 'type'), work_form(type), obj, mvs)
110
+ elsif type.include? "File"
111
+ metadata = create_data(row.except('id', 'type'), file_form, obj, mvs)
112
+ end
113
+ unless metadata.nil?
114
+ obj.attributes = metadata
115
+ obj.try(:to_controlled_vocab)
116
+ obj.save
117
+ end
118
+ end
119
+ flash[:notice] = "csv successfully uploaded"
120
+ redirect_to csv_edit_path
121
+ end
122
+
123
+ def export
124
+ csv_export_params
125
+ # Get a collection's member works from Solr
126
+ solr = RSolr.connect url: Blacklight.connection_config[:url]
127
+ response = solr.get 'select', params: {
128
+ q: "member_of_collection_ids_ssim:#{params[:csv_export][:collection]}",
129
+ fq: ["has_model_ssim:FileSet OR has_model_ssim:*Work"],
130
+ rows: 3400,
131
+ fl: "id"
132
+ }
133
+ unless response['response']['docs'].empty? || response['response']['docs'][0].empty?
134
+ work_ids = response['response']['docs'].map { |doc| doc['id'] }
135
+ end
136
+
137
+ send_data CsvExportService.new(available_works).csv_for(work_ids),
138
+ :type => 'text/csv; charset=iso-8859-5; header=present',
139
+ :disposition => "attachment; filename=export.csv"
140
+ end
141
+
142
+ private
143
+
144
+ def authenticate
145
+ authorize! :create, available_works.first
146
+ end
147
+
148
+ def csv_upload_params
149
+ params.require(:csv_import).permit(:csv_file, :mvs, :admin_set, :collection)
150
+ end
151
+
152
+ def csv_export_params
153
+ params.require(:csv_export).permit(:collection)
154
+ end
155
+
156
+ def available_works
157
+ @available_works ||= Hyrax::QuickClassificationQuery.new(current_user).authorized_models
158
+ end
159
+
160
+ def parse_csv csv, mvs
161
+ csv = CSV.parse(File.read(csv), headers: true, encoding: 'utf-8').map(&:to_hash)
162
+ @works = []
163
+ csv.each do |row|
164
+ type = row.first.last
165
+ if type.nil?
166
+ next
167
+ elsif type.include? "Work"
168
+ metadata = create_data(row, work_form(type), Object.const_get(type).new, mvs)
169
+ @works << {type: type, metadata: metadata, files: []}
170
+ elsif type.include? "File"
171
+ metadata = create_data(row, file_form, FileSet.new, mvs)
172
+ @works.last[:files] << {url: row.delete('url'), title: row.delete('title'), metadata: metadata}
173
+ end
174
+ end
175
+ end
176
+
177
+ def load_config
178
+ # multitenant? defined in ApplicationController
179
+ if multitenant?
180
+ tenant = Account.find_by(tenant: Apartment::Tenant.current).cname
181
+ else
182
+ tenant = "default"
183
+ end
184
+ if CdmMigrator::Engine.config['tenant_settings'].has_key?(tenant)
185
+ settings = CdmMigrator::Engine.config['tenant_settings'][tenant]['csv_checker']
186
+ if settings.present?
187
+ @date_indexing_service = settings['date_indexing_service'].first.constantize if settings['date_indexing_service']
188
+ @date_fields = settings['date_fields'].map(&:to_sym) if settings['date_fields']
189
+ @uri_fields = settings['valid_uri_fields'].map(&:to_sym) if settings['valid_uri_fields']
190
+ @separator = settings['multi_value_separator']
191
+ @separator_fields = settings['separator_fields'].map(&:to_sym) if settings['separator_fields']
192
+ @path_to_drive = settings['path_to_drive']
193
+ # If you would like to change this to match the uploader's max file size,
194
+ # change this to Hyrax.config.uploader[:maxFileSize]
195
+ @max_file_size = settings['max_file_size']
196
+ else
197
+ raise "Cdm Migrator couldn't find any configured settings. Are they in cdm_migrator.yml?"
198
+ end
199
+ else
200
+ raise "Cdm Migrator couldn't find this tenant. Is it configured?"
201
+ end
202
+ end
203
+
204
+ def check_csv csv_file
205
+ results = CsvChecker.new.check_csv(csv_file)
206
+ @error_list = results['errors']
207
+ end
208
+
209
+ def default_page_title
210
+ 'CSV Batch Uploader'
211
+ end
212
+
213
+ def admin_host?
214
+ # multitenant? defined in ApplicationController
215
+ false unless multitenant?
216
+ end
217
+
218
+ def available_translations
219
+ {
220
+ 'en' => 'English',
221
+ 'fr' => 'French'
222
+ }
223
+ end
224
+
225
+ def work_form(worktype = "GenericWork")
226
+ Module.const_get("Hyrax::#{worktype}Form") rescue nil || Module.const_get("Hyrax::Forms::WorkForm")
227
+ end
228
+
229
+ def file_form
230
+ Module.const_get("Hyrax::FileSetForm") rescue nil || Module.const_get("Hyrax::Forms::FileSetEditForm")
231
+ end
232
+
233
+ def secondary_terms form_name
234
+ form_name.terms - form_name.required_fields -
235
+ [:visibility_during_embargo, :embargo_release_date,
236
+ :visibility_after_embargo, :visibility_during_lease,
237
+ :lease_expiration_date, :visibility_after_lease, :visibility,
238
+ :thumbnail_id, :representative_id, :ordered_member_ids,
239
+ :collection_ids, :in_works_ids, :admin_set_id, :files, :source, :member_of_collection_ids]
240
+ end
241
+
242
+ def create_data data, type, object, mvs
243
+ final_data = {}
244
+ accepted_terms = type.required_fields + secondary_terms(type)
245
+ data.each do |key, att|
246
+ if (att.nil? || att.empty? || key.to_s.include?("object_type") || !accepted_terms.include?(key.to_sym))
247
+ next
248
+ elsif object.send(key).nil?
249
+ final_data[key] = att
250
+ else
251
+ if object.class.properties[key.to_s].multiple?
252
+ final_data[key] = att.split(mvs)
253
+ else
254
+ final_data[key] = att
255
+ end
256
+ end
257
+ end
258
+ final_data
259
+ end
260
+
261
+ def create_lease visibility, status_after, date
262
+ lease = Hydra::AccessControls::Lease.new(visibility_during_lease: visibility,
263
+ visibility_after_lease: status_after, lease_expiration_date: @lease_date)
264
+ lease.save
265
+ end
266
+
267
+ def create_embargo visibility
268
+ embargo = Hydra::AccessControls::Embargo.new
269
+ embargo.visibility_during_embargo = visibility
270
+ embargo.visibility_after_embargo = @status_after
271
+ embargo.embargo_release_date = @embargo_date
272
+ embargo.save
273
+ end
274
+
275
+ def log(user)
276
+ Hyrax::Operation.create!(user: user,
277
+ operation_type: "Attach Remote File")
278
+ end
279
+ end
382
280
  end