bulkrax 9.3.5 → 9.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +11 -1
- data/app/assets/javascripts/bulkrax/application.js +2 -1
- data/app/assets/javascripts/bulkrax/bulkrax.js +13 -4
- data/app/assets/javascripts/bulkrax/bulkrax_utils.js +96 -0
- data/app/assets/javascripts/bulkrax/datatables.js +1 -0
- data/app/assets/javascripts/bulkrax/entries.js +17 -10
- data/app/assets/javascripts/bulkrax/importers.js.erb +9 -2
- data/app/assets/javascripts/bulkrax/importers_stepper.js +2420 -0
- data/app/assets/stylesheets/bulkrax/application.css +1 -1
- data/app/assets/stylesheets/bulkrax/stepper/_header.scss +83 -0
- data/app/assets/stylesheets/bulkrax/stepper/_mixins.scss +26 -0
- data/app/assets/stylesheets/bulkrax/stepper/_navigation.scss +103 -0
- data/app/assets/stylesheets/bulkrax/stepper/_responsive.scss +46 -0
- data/app/assets/stylesheets/bulkrax/stepper/_review.scss +92 -0
- data/app/assets/stylesheets/bulkrax/stepper/_settings.scss +106 -0
- data/app/assets/stylesheets/bulkrax/stepper/_success.scss +26 -0
- data/app/assets/stylesheets/bulkrax/stepper/_summary.scss +171 -0
- data/app/assets/stylesheets/bulkrax/stepper/_upload.scss +339 -0
- data/app/assets/stylesheets/bulkrax/stepper/_validation.scss +237 -0
- data/app/assets/stylesheets/bulkrax/stepper/_variables.scss +46 -0
- data/app/assets/stylesheets/bulkrax/stepper.scss +32 -0
- data/app/controllers/bulkrax/guided_imports_controller.rb +175 -0
- data/app/controllers/bulkrax/importers_controller.rb +28 -31
- data/app/controllers/concerns/bulkrax/guided_import_demo_scenarios.rb +201 -0
- data/app/controllers/concerns/bulkrax/importer_file_handler.rb +217 -0
- data/app/factories/bulkrax/object_factory.rb +3 -2
- data/app/factories/bulkrax/valkyrie_object_factory.rb +61 -17
- data/app/jobs/bulkrax/importer_job.rb +11 -4
- data/app/models/bulkrax/csv_entry.rb +27 -7
- data/app/models/bulkrax/entry.rb +4 -0
- data/app/models/bulkrax/importer.rb +31 -1
- data/app/models/concerns/bulkrax/has_matchers.rb +2 -2
- data/app/models/concerns/bulkrax/importer_exporter_behavior.rb +6 -5
- data/app/parsers/bulkrax/application_parser.rb +31 -5
- data/app/parsers/bulkrax/csv_parser.rb +42 -10
- data/app/parsers/concerns/bulkrax/csv_parser/csv_template_generation.rb +73 -0
- data/app/parsers/concerns/bulkrax/csv_parser/csv_validation.rb +133 -0
- data/app/parsers/concerns/bulkrax/csv_parser/csv_validation_helpers.rb +282 -0
- data/app/parsers/concerns/bulkrax/csv_parser/csv_validation_hierarchy.rb +96 -0
- data/app/services/bulkrax/csv_template/column_builder.rb +60 -0
- data/app/services/bulkrax/csv_template/column_descriptor.rb +58 -0
- data/app/services/bulkrax/csv_template/csv_builder.rb +83 -0
- data/app/services/bulkrax/csv_template/explanation_builder.rb +57 -0
- data/app/services/bulkrax/csv_template/field_analyzer.rb +56 -0
- data/app/services/bulkrax/csv_template/file_path_generator.rb +47 -0
- data/app/services/bulkrax/csv_template/file_validator.rb +68 -0
- data/app/services/bulkrax/csv_template/mapping_manager.rb +55 -0
- data/app/services/bulkrax/csv_template/model_loader.rb +50 -0
- data/app/services/bulkrax/csv_template/row_builder.rb +35 -0
- data/app/services/bulkrax/csv_template/schema_analyzer.rb +70 -0
- data/app/services/bulkrax/csv_template/split_formatter.rb +44 -0
- data/app/services/bulkrax/csv_template/value_determiner.rb +68 -0
- data/app/services/bulkrax/stepper_response_formatter.rb +347 -0
- data/app/services/bulkrax/validation_error_csv_builder.rb +99 -0
- data/app/validators/bulkrax/csv_row/child_reference.rb +56 -0
- data/app/validators/bulkrax/csv_row/circular_reference.rb +71 -0
- data/app/validators/bulkrax/csv_row/controlled_vocabulary.rb +74 -0
- data/app/validators/bulkrax/csv_row/duplicate_identifier.rb +63 -0
- data/app/validators/bulkrax/csv_row/missing_source_identifier.rb +31 -0
- data/app/validators/bulkrax/csv_row/parent_reference.rb +59 -0
- data/app/validators/bulkrax/csv_row/required_values.rb +64 -0
- data/app/views/bulkrax/guided_imports/new.html.erb +567 -0
- data/app/views/bulkrax/importers/index.html.erb +6 -1
- data/app/views/bulkrax/importers/new.html.erb +1 -1
- data/app/views/bulkrax/importers/show.html.erb +17 -1
- data/config/i18n-tasks.yml +195 -0
- data/config/locales/bulkrax.de.yml +504 -0
- data/config/locales/bulkrax.en.yml +459 -233
- data/config/locales/bulkrax.es.yml +504 -0
- data/config/locales/bulkrax.fr.yml +504 -0
- data/config/locales/bulkrax.it.yml +504 -0
- data/config/locales/bulkrax.pt-BR.yml +504 -0
- data/config/locales/bulkrax.zh.yml +503 -0
- data/config/routes.rb +10 -1
- data/lib/bulkrax/data/demo_scenarios.json +2235 -0
- data/lib/bulkrax/version.rb +1 -1
- data/lib/bulkrax.rb +31 -0
- metadata +55 -16
- data/app/services/bulkrax/sample_csv_service/column_builder.rb +0 -58
- data/app/services/bulkrax/sample_csv_service/column_descriptor.rb +0 -56
- data/app/services/bulkrax/sample_csv_service/csv_builder.rb +0 -82
- data/app/services/bulkrax/sample_csv_service/explanation_builder.rb +0 -51
- data/app/services/bulkrax/sample_csv_service/field_analyzer.rb +0 -54
- data/app/services/bulkrax/sample_csv_service/file_path_generator.rb +0 -16
- data/app/services/bulkrax/sample_csv_service/mapping_manager.rb +0 -36
- data/app/services/bulkrax/sample_csv_service/model_loader.rb +0 -40
- data/app/services/bulkrax/sample_csv_service/row_builder.rb +0 -33
- data/app/services/bulkrax/sample_csv_service/schema_analyzer.rb +0 -69
- data/app/services/bulkrax/sample_csv_service/split_formatter.rb +0 -42
- data/app/services/bulkrax/sample_csv_service/value_determiner.rb +0 -67
- data/app/services/bulkrax/sample_csv_service.rb +0 -78
- /data/{app/services → lib}/wings/custom_queries/find_by_source_identifier.rb +0 -0
data/lib/bulkrax/version.rb
CHANGED
data/lib/bulkrax.rb
CHANGED
|
@@ -15,6 +15,8 @@ require 'nokogiri'
|
|
|
15
15
|
require 'ostruct'
|
|
16
16
|
require 'zip'
|
|
17
17
|
|
|
18
|
+
require 'wings/custom_queries/find_by_source_identifier'
|
|
19
|
+
|
|
18
20
|
def conditional_require(gem_name)
|
|
19
21
|
require gem_name
|
|
20
22
|
rescue LoadError
|
|
@@ -37,6 +39,8 @@ module Bulkrax
|
|
|
37
39
|
:default_work_type,
|
|
38
40
|
:export_path,
|
|
39
41
|
:field_mappings,
|
|
42
|
+
:guided_import_enabled,
|
|
43
|
+
:guided_import_demo_scenarios_enabled,
|
|
40
44
|
:generated_metadata_mapping,
|
|
41
45
|
:import_path,
|
|
42
46
|
:multi_value_element_join_on,
|
|
@@ -170,6 +174,28 @@ module Bulkrax
|
|
|
170
174
|
ENV.key?("REDIS_HOST")
|
|
171
175
|
end
|
|
172
176
|
alias use_locking? use_locking
|
|
177
|
+
|
|
178
|
+
##
|
|
179
|
+
# @return [Array<#call>] callable validators invoked per-row during CSV validation.
|
|
180
|
+
# Each callable receives (record, row_number, context).
|
|
181
|
+
# Defaults to the four built-in CsvRow:: validators.
|
|
182
|
+
def csv_row_validators
|
|
183
|
+
@csv_row_validators ||= [
|
|
184
|
+
Bulkrax::CsvRow::MissingSourceIdentifier,
|
|
185
|
+
Bulkrax::CsvRow::DuplicateIdentifier,
|
|
186
|
+
Bulkrax::CsvRow::ParentReference,
|
|
187
|
+
Bulkrax::CsvRow::ChildReference,
|
|
188
|
+
Bulkrax::CsvRow::CircularReference,
|
|
189
|
+
Bulkrax::CsvRow::RequiredValues,
|
|
190
|
+
Bulkrax::CsvRow::ControlledVocabulary
|
|
191
|
+
]
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
attr_writer :csv_row_validators
|
|
195
|
+
|
|
196
|
+
def register_csv_row_validator(callable)
|
|
197
|
+
csv_row_validators << callable
|
|
198
|
+
end
|
|
173
199
|
end
|
|
174
200
|
|
|
175
201
|
def config
|
|
@@ -226,6 +252,9 @@ module Bulkrax
|
|
|
226
252
|
:required_elements=,
|
|
227
253
|
:reserved_properties,
|
|
228
254
|
:reserved_properties=,
|
|
255
|
+
:csv_row_validators,
|
|
256
|
+
:csv_row_validators=,
|
|
257
|
+
:register_csv_row_validator,
|
|
229
258
|
:server_name,
|
|
230
259
|
:server_name=,
|
|
231
260
|
:solr_key_for_member_file_ids,
|
|
@@ -249,6 +278,8 @@ module Bulkrax
|
|
|
249
278
|
conf.server_name = 'bulkrax@example.com'
|
|
250
279
|
conf.relationship_job_class = "Bulkrax::CreateRelationshipsJob"
|
|
251
280
|
conf.required_elements = ['title']
|
|
281
|
+
conf.guided_import_enabled = ActiveModel::Type::Boolean.new.cast(ENV.fetch('BULKRAX_GUIDED_IMPORTER', false))
|
|
282
|
+
conf.guided_import_demo_scenarios_enabled = false
|
|
252
283
|
|
|
253
284
|
# Hash of Generic field_mappings for use in the view
|
|
254
285
|
# There must be one field_mappings hash per view partial
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: bulkrax
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 9.
|
|
4
|
+
version: 9.4.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Rob Kaufman
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-
|
|
11
|
+
date: 2026-04-15 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: rails
|
|
@@ -331,23 +331,40 @@ files:
|
|
|
331
331
|
- app/assets/config/bulkrax_manifest.js
|
|
332
332
|
- app/assets/javascripts/bulkrax/application.js
|
|
333
333
|
- app/assets/javascripts/bulkrax/bulkrax.js
|
|
334
|
+
- app/assets/javascripts/bulkrax/bulkrax_utils.js
|
|
334
335
|
- app/assets/javascripts/bulkrax/datatables.js
|
|
335
336
|
- app/assets/javascripts/bulkrax/entries.js
|
|
336
337
|
- app/assets/javascripts/bulkrax/exporters.js
|
|
337
338
|
- app/assets/javascripts/bulkrax/importers.js.erb
|
|
339
|
+
- app/assets/javascripts/bulkrax/importers_stepper.js
|
|
338
340
|
- app/assets/javascripts/bulkrax/navtabs.js.erb
|
|
339
341
|
- app/assets/stylesheets/bulkrax/accordion.scss
|
|
340
342
|
- app/assets/stylesheets/bulkrax/application.css
|
|
341
343
|
- app/assets/stylesheets/bulkrax/coderay.scss
|
|
342
344
|
- app/assets/stylesheets/bulkrax/import_export.scss
|
|
345
|
+
- app/assets/stylesheets/bulkrax/stepper.scss
|
|
346
|
+
- app/assets/stylesheets/bulkrax/stepper/_header.scss
|
|
347
|
+
- app/assets/stylesheets/bulkrax/stepper/_mixins.scss
|
|
348
|
+
- app/assets/stylesheets/bulkrax/stepper/_navigation.scss
|
|
349
|
+
- app/assets/stylesheets/bulkrax/stepper/_responsive.scss
|
|
350
|
+
- app/assets/stylesheets/bulkrax/stepper/_review.scss
|
|
351
|
+
- app/assets/stylesheets/bulkrax/stepper/_settings.scss
|
|
352
|
+
- app/assets/stylesheets/bulkrax/stepper/_success.scss
|
|
353
|
+
- app/assets/stylesheets/bulkrax/stepper/_summary.scss
|
|
354
|
+
- app/assets/stylesheets/bulkrax/stepper/_upload.scss
|
|
355
|
+
- app/assets/stylesheets/bulkrax/stepper/_validation.scss
|
|
356
|
+
- app/assets/stylesheets/bulkrax/stepper/_variables.scss
|
|
343
357
|
- app/concerns/loggable.rb
|
|
344
358
|
- app/controllers/bulkrax/application_controller.rb
|
|
345
359
|
- app/controllers/bulkrax/entries_controller.rb
|
|
346
360
|
- app/controllers/bulkrax/exporters_controller.rb
|
|
361
|
+
- app/controllers/bulkrax/guided_imports_controller.rb
|
|
347
362
|
- app/controllers/bulkrax/importers_controller.rb
|
|
348
363
|
- app/controllers/concerns/bulkrax/api.rb
|
|
349
364
|
- app/controllers/concerns/bulkrax/datatables_behavior.rb
|
|
350
365
|
- app/controllers/concerns/bulkrax/download_behavior.rb
|
|
366
|
+
- app/controllers/concerns/bulkrax/guided_import_demo_scenarios.rb
|
|
367
|
+
- app/controllers/concerns/bulkrax/importer_file_handler.rb
|
|
351
368
|
- app/factories/bulkrax/object_factory.rb
|
|
352
369
|
- app/factories/bulkrax/object_factory_interface.rb
|
|
353
370
|
- app/factories/bulkrax/valkyrie_object_factory.rb
|
|
@@ -416,23 +433,35 @@ files:
|
|
|
416
433
|
- app/parsers/bulkrax/oai_qualified_dc_parser.rb
|
|
417
434
|
- app/parsers/bulkrax/parser_export_record_set.rb
|
|
418
435
|
- app/parsers/bulkrax/xml_parser.rb
|
|
436
|
+
- app/parsers/concerns/bulkrax/csv_parser/csv_template_generation.rb
|
|
437
|
+
- app/parsers/concerns/bulkrax/csv_parser/csv_validation.rb
|
|
438
|
+
- app/parsers/concerns/bulkrax/csv_parser/csv_validation_helpers.rb
|
|
439
|
+
- app/parsers/concerns/bulkrax/csv_parser/csv_validation_hierarchy.rb
|
|
440
|
+
- app/services/bulkrax/csv_template/column_builder.rb
|
|
441
|
+
- app/services/bulkrax/csv_template/column_descriptor.rb
|
|
442
|
+
- app/services/bulkrax/csv_template/csv_builder.rb
|
|
443
|
+
- app/services/bulkrax/csv_template/explanation_builder.rb
|
|
444
|
+
- app/services/bulkrax/csv_template/field_analyzer.rb
|
|
445
|
+
- app/services/bulkrax/csv_template/file_path_generator.rb
|
|
446
|
+
- app/services/bulkrax/csv_template/file_validator.rb
|
|
447
|
+
- app/services/bulkrax/csv_template/mapping_manager.rb
|
|
448
|
+
- app/services/bulkrax/csv_template/model_loader.rb
|
|
449
|
+
- app/services/bulkrax/csv_template/row_builder.rb
|
|
450
|
+
- app/services/bulkrax/csv_template/schema_analyzer.rb
|
|
451
|
+
- app/services/bulkrax/csv_template/split_formatter.rb
|
|
452
|
+
- app/services/bulkrax/csv_template/value_determiner.rb
|
|
419
453
|
- app/services/bulkrax/factory_class_finder.rb
|
|
420
454
|
- app/services/bulkrax/remove_relationships_for_importer.rb
|
|
421
|
-
- app/services/bulkrax/
|
|
422
|
-
- app/services/bulkrax/
|
|
423
|
-
- app/services/bulkrax/sample_csv_service/column_descriptor.rb
|
|
424
|
-
- app/services/bulkrax/sample_csv_service/csv_builder.rb
|
|
425
|
-
- app/services/bulkrax/sample_csv_service/explanation_builder.rb
|
|
426
|
-
- app/services/bulkrax/sample_csv_service/field_analyzer.rb
|
|
427
|
-
- app/services/bulkrax/sample_csv_service/file_path_generator.rb
|
|
428
|
-
- app/services/bulkrax/sample_csv_service/mapping_manager.rb
|
|
429
|
-
- app/services/bulkrax/sample_csv_service/model_loader.rb
|
|
430
|
-
- app/services/bulkrax/sample_csv_service/row_builder.rb
|
|
431
|
-
- app/services/bulkrax/sample_csv_service/schema_analyzer.rb
|
|
432
|
-
- app/services/bulkrax/sample_csv_service/split_formatter.rb
|
|
433
|
-
- app/services/bulkrax/sample_csv_service/value_determiner.rb
|
|
455
|
+
- app/services/bulkrax/stepper_response_formatter.rb
|
|
456
|
+
- app/services/bulkrax/validation_error_csv_builder.rb
|
|
434
457
|
- app/services/hyrax/custom_queries/find_by_source_identifier.rb
|
|
435
|
-
- app/
|
|
458
|
+
- app/validators/bulkrax/csv_row/child_reference.rb
|
|
459
|
+
- app/validators/bulkrax/csv_row/circular_reference.rb
|
|
460
|
+
- app/validators/bulkrax/csv_row/controlled_vocabulary.rb
|
|
461
|
+
- app/validators/bulkrax/csv_row/duplicate_identifier.rb
|
|
462
|
+
- app/validators/bulkrax/csv_row/missing_source_identifier.rb
|
|
463
|
+
- app/validators/bulkrax/csv_row/parent_reference.rb
|
|
464
|
+
- app/validators/bulkrax/csv_row/required_values.rb
|
|
436
465
|
- app/views/bulkrax/entries/_parsed_metadata.html.erb
|
|
437
466
|
- app/views/bulkrax/entries/_raw_metadata.html.erb
|
|
438
467
|
- app/views/bulkrax/entries/show.html.erb
|
|
@@ -442,6 +471,7 @@ files:
|
|
|
442
471
|
- app/views/bulkrax/exporters/index.html.erb
|
|
443
472
|
- app/views/bulkrax/exporters/new.html.erb
|
|
444
473
|
- app/views/bulkrax/exporters/show.html.erb
|
|
474
|
+
- app/views/bulkrax/guided_imports/new.html.erb
|
|
445
475
|
- app/views/bulkrax/importers/_bagit_fields.html.erb
|
|
446
476
|
- app/views/bulkrax/importers/_browse_everything.html.erb
|
|
447
477
|
- app/views/bulkrax/importers/_csv_fields.html.erb
|
|
@@ -462,7 +492,14 @@ files:
|
|
|
462
492
|
- app/views/hyrax/dashboard/sidebar/_bulkrax_sidebar_additions.html.erb
|
|
463
493
|
- app/views/hyrax/dashboard/sidebar/_repository_content.html.erb
|
|
464
494
|
- app/views/layouts/bulkrax/application.html.erb
|
|
495
|
+
- config/i18n-tasks.yml
|
|
496
|
+
- config/locales/bulkrax.de.yml
|
|
465
497
|
- config/locales/bulkrax.en.yml
|
|
498
|
+
- config/locales/bulkrax.es.yml
|
|
499
|
+
- config/locales/bulkrax.fr.yml
|
|
500
|
+
- config/locales/bulkrax.it.yml
|
|
501
|
+
- config/locales/bulkrax.pt-BR.yml
|
|
502
|
+
- config/locales/bulkrax.zh.yml
|
|
466
503
|
- config/routes.rb
|
|
467
504
|
- db/migrate/20181011230201_create_bulkrax_importers.rb
|
|
468
505
|
- db/migrate/20181011230228_create_bulkrax_importer_runs.rb
|
|
@@ -509,6 +546,7 @@ files:
|
|
|
509
546
|
- db/migrate/20241203010707_entry_error_denormalization.rb
|
|
510
547
|
- db/migrate/20241205212513_faster_first_entry.rb
|
|
511
548
|
- lib/bulkrax.rb
|
|
549
|
+
- lib/bulkrax/data/demo_scenarios.json
|
|
512
550
|
- lib/bulkrax/engine.rb
|
|
513
551
|
- lib/bulkrax/entry_spec_helper.rb
|
|
514
552
|
- lib/bulkrax/version.rb
|
|
@@ -521,6 +559,7 @@ files:
|
|
|
521
559
|
- lib/generators/bulkrax/templates/config/initializers/bulkrax.rb
|
|
522
560
|
- lib/tasks/bulkrax_tasks.rake
|
|
523
561
|
- lib/tasks/reset.rake
|
|
562
|
+
- lib/wings/custom_queries/find_by_source_identifier.rb
|
|
524
563
|
homepage: https://github.com/samvera-labs/bulkrax
|
|
525
564
|
licenses:
|
|
526
565
|
- Apache-2.0
|
|
@@ -1,58 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Bulkrax
|
|
4
|
-
# Builds column headers for CSV
|
|
5
|
-
class SampleCsvService::ColumnBuilder
|
|
6
|
-
def initialize(service)
|
|
7
|
-
@service = service
|
|
8
|
-
@descriptor = SampleCsvService::ColumnDescriptor.new
|
|
9
|
-
end
|
|
10
|
-
|
|
11
|
-
def all_columns
|
|
12
|
-
required_columns + property_columns
|
|
13
|
-
end
|
|
14
|
-
|
|
15
|
-
def required_columns
|
|
16
|
-
mapped_core_columns +
|
|
17
|
-
relationship_columns +
|
|
18
|
-
file_columns
|
|
19
|
-
end
|
|
20
|
-
|
|
21
|
-
private
|
|
22
|
-
|
|
23
|
-
def mapped_core_columns
|
|
24
|
-
@descriptor.core_columns.map do |column|
|
|
25
|
-
@service.mapping_manager.key_to_mapped_column(column)
|
|
26
|
-
end
|
|
27
|
-
end
|
|
28
|
-
|
|
29
|
-
def property_columns
|
|
30
|
-
field_lists = @service.all_models.map do |m|
|
|
31
|
-
@service.field_analyzer.find_or_create_field_list_for(model_name: m)
|
|
32
|
-
end
|
|
33
|
-
|
|
34
|
-
properties = field_lists
|
|
35
|
-
.flat_map { |item| item.values.flat_map { |config| config["properties"] || [] } }
|
|
36
|
-
.uniq
|
|
37
|
-
.map { |property| @service.mapping_manager.key_to_mapped_column(property) }
|
|
38
|
-
.uniq
|
|
39
|
-
|
|
40
|
-
(properties - required_columns).sort
|
|
41
|
-
end
|
|
42
|
-
|
|
43
|
-
def relationship_columns
|
|
44
|
-
[
|
|
45
|
-
@service.mapping_manager.find_by_flag("related_children_field_mapping", 'children'),
|
|
46
|
-
@service.mapping_manager.find_by_flag("related_parents_field_mapping", 'parents')
|
|
47
|
-
]
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
def file_columns
|
|
51
|
-
SampleCsvService::ColumnDescriptor::COLUMN_DESCRIPTIONS[:files].flat_map do |property_hash|
|
|
52
|
-
property_hash.keys.map do |key|
|
|
53
|
-
@service.mapping_manager.key_to_mapped_column(key)
|
|
54
|
-
end
|
|
55
|
-
end
|
|
56
|
-
end
|
|
57
|
-
end
|
|
58
|
-
end
|
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Bulkrax
|
|
4
|
-
# Manages column descriptions and metadata
|
|
5
|
-
class SampleCsvService::ColumnDescriptor
|
|
6
|
-
COLUMN_DESCRIPTIONS = {
|
|
7
|
-
include_first: [
|
|
8
|
-
{ "model" => "The work types configured in your repository are listed below.\nIf left blank, your default work type, #{Bulkrax.default_work_type}, is used." },
|
|
9
|
-
{ "source_identifier" => "This must be a unique identifier.\nIt can be alphanumeric with some special characters (e.g. hyphens, colons), and URLs are also supported." },
|
|
10
|
-
{ "id" => "This column would optionally be included only if it is a re-import, i.e. for updating or deleting records.\nThis is a key identifier used by the system, which you wouldn't have for new imports." },
|
|
11
|
-
{ "rights_statement" => "Rights statement URI for the work.\nIf not included, uses the value specified on the bulk import configuration screen." }
|
|
12
|
-
],
|
|
13
|
-
visibility: [
|
|
14
|
-
{ "visibility" => "Uses the value specified on the bulk import configuration screen if not added here.\nValid options: open, authenticated, restricted, embargo, lease" },
|
|
15
|
-
{ "embargo_release_date" => "Required for embargo (yyyy-mm-dd)" },
|
|
16
|
-
{ "visibility_during_embargo" => "Required for embargo" },
|
|
17
|
-
{ "visibility_after_embargo" => "Required for embargo" },
|
|
18
|
-
{ "lease_expiration_date" => "Required for lease (yyyy-mm-dd)" },
|
|
19
|
-
{ "visibility_during_lease" => "Required for lease" },
|
|
20
|
-
{ "visibility_after_lease" => "Required for lease" }
|
|
21
|
-
],
|
|
22
|
-
files: [
|
|
23
|
-
{ "file" => "Use filenames exactly matching those in your files folder.\nZip your CSV and files folder together and attach this to your importer." },
|
|
24
|
-
{ "remote_files" => "Use the URLs to remote files to be attached to the work." }
|
|
25
|
-
],
|
|
26
|
-
relationships: [
|
|
27
|
-
{ "parents" => "The source_identifier or id of work or collection to be attached as parent." },
|
|
28
|
-
{ "children" => "The source_identifier or id of work or file to be attached as child." }
|
|
29
|
-
],
|
|
30
|
-
other: [
|
|
31
|
-
{ "hide_from_catalog_search" => "Set to 1 to hide the collection from catalog search results." },
|
|
32
|
-
{ "show_pdf_download_button" => "Set to 1 to show a PDF download link on the work's page." },
|
|
33
|
-
{ "show_pdf_viewer" => "Set to 1 to show a PDF viewer on the work's page." },
|
|
34
|
-
{ "video_embed" => "A valid URL to a hosted video that can appear in an iframe, beginning with 'http://' or 'https://'." }
|
|
35
|
-
]
|
|
36
|
-
}.freeze
|
|
37
|
-
|
|
38
|
-
def core_columns
|
|
39
|
-
extract_column_names(:include_first) + extract_column_names(:visibility)
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
def find_description_for(column)
|
|
43
|
-
COLUMN_DESCRIPTIONS.each_value do |group|
|
|
44
|
-
prop = group.find { |hash| hash.key?(column) }
|
|
45
|
-
return prop[column] if prop
|
|
46
|
-
end
|
|
47
|
-
nil
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
private
|
|
51
|
-
|
|
52
|
-
def extract_column_names(group)
|
|
53
|
-
COLUMN_DESCRIPTIONS[group].map { |hash| hash.keys.first }
|
|
54
|
-
end
|
|
55
|
-
end
|
|
56
|
-
end
|
|
@@ -1,82 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Bulkrax
|
|
4
|
-
# Builds CSV content
|
|
5
|
-
class SampleCsvService::CsvBuilder
|
|
6
|
-
IGNORED_PROPERTIES = %w[
|
|
7
|
-
admin_set_id alternate_ids
|
|
8
|
-
bulkrax_identifier
|
|
9
|
-
collection_type_gid contexts created_at
|
|
10
|
-
date date_modified date_uploaded depositor
|
|
11
|
-
embargo embargo_id
|
|
12
|
-
file_ids
|
|
13
|
-
has_model head
|
|
14
|
-
internal_resource is_child
|
|
15
|
-
lease lease_id
|
|
16
|
-
member_ids member_of_collection_ids modified_date
|
|
17
|
-
new_record
|
|
18
|
-
on_behalf_of owner proxy_depositor
|
|
19
|
-
rendering_ids representative_id
|
|
20
|
-
schema_version split_from_pdf_id state tail
|
|
21
|
-
thumbnail_id
|
|
22
|
-
updated_at
|
|
23
|
-
].freeze
|
|
24
|
-
|
|
25
|
-
def initialize(service)
|
|
26
|
-
@service = service
|
|
27
|
-
@column_builder = SampleCsvService::ColumnBuilder.new(service)
|
|
28
|
-
@row_builder = SampleCsvService::RowBuilder.new(service)
|
|
29
|
-
@header_row = nil
|
|
30
|
-
@required_headings = []
|
|
31
|
-
end
|
|
32
|
-
|
|
33
|
-
def write_to_file(file_path)
|
|
34
|
-
FileUtils.mkdir_p(File.dirname(file_path))
|
|
35
|
-
CSV.open(file_path, "w") { |csv| write_rows(csv) }
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
def generate_string
|
|
39
|
-
CSV.generate { |csv| write_rows(csv) }
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
private
|
|
43
|
-
|
|
44
|
-
def write_rows(csv)
|
|
45
|
-
csv_rows.each { |row| csv << row }
|
|
46
|
-
end
|
|
47
|
-
|
|
48
|
-
def csv_rows
|
|
49
|
-
@header_row = fill_header_row
|
|
50
|
-
rows = [
|
|
51
|
-
@header_row,
|
|
52
|
-
@row_builder.build_explanation_row(@header_row),
|
|
53
|
-
*@row_builder.build_model_rows(@header_row)
|
|
54
|
-
]
|
|
55
|
-
remove_empty_columns(rows)
|
|
56
|
-
end
|
|
57
|
-
|
|
58
|
-
def fill_header_row
|
|
59
|
-
@required_headings = @column_builder.required_columns
|
|
60
|
-
all_columns = @column_builder.all_columns
|
|
61
|
-
filtered = all_columns - IGNORED_PROPERTIES
|
|
62
|
-
@required_headings = @column_builder.required_columns & filtered
|
|
63
|
-
filtered
|
|
64
|
-
end
|
|
65
|
-
|
|
66
|
-
def remove_empty_columns(rows)
|
|
67
|
-
return rows if rows.empty?
|
|
68
|
-
|
|
69
|
-
columns = rows.transpose
|
|
70
|
-
non_empty_columns = columns.select { |col| keep_column?(col) }
|
|
71
|
-
non_empty_columns.transpose
|
|
72
|
-
end
|
|
73
|
-
|
|
74
|
-
def keep_column?(column)
|
|
75
|
-
heading = column[0]
|
|
76
|
-
return true if @required_headings.include?(heading)
|
|
77
|
-
|
|
78
|
-
# Check if any data row has content
|
|
79
|
-
column[2..-1].any? { |value| !value.nil? && value != "" && value != "---" }
|
|
80
|
-
end
|
|
81
|
-
end
|
|
82
|
-
end
|
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Bulkrax
|
|
4
|
-
# Builds explanations for CSV columns
|
|
5
|
-
class SampleCsvService::ExplanationBuilder
|
|
6
|
-
def initialize(service)
|
|
7
|
-
@service = service
|
|
8
|
-
@descriptor = SampleCsvService::ColumnDescriptor.new
|
|
9
|
-
@split_formatter = SampleCsvService::SplitFormatter.new
|
|
10
|
-
end
|
|
11
|
-
|
|
12
|
-
def build_explanations(header_row)
|
|
13
|
-
header_row.map do |column|
|
|
14
|
-
{ column => build_explanation(column) }
|
|
15
|
-
end
|
|
16
|
-
end
|
|
17
|
-
|
|
18
|
-
private
|
|
19
|
-
|
|
20
|
-
def build_explanation(column)
|
|
21
|
-
mapping_key = @service.mapping_manager.mapped_to_key(column)
|
|
22
|
-
|
|
23
|
-
column_description = @descriptor.find_description_for(column)
|
|
24
|
-
controlled_vocab_info = controlled_vocab_text(mapping_key)
|
|
25
|
-
split_info = split_text(mapping_key, controlled_vocab_info)
|
|
26
|
-
|
|
27
|
-
components = [
|
|
28
|
-
column_description,
|
|
29
|
-
controlled_vocab_info,
|
|
30
|
-
split_info
|
|
31
|
-
].compact
|
|
32
|
-
|
|
33
|
-
components.join("\n")
|
|
34
|
-
end
|
|
35
|
-
|
|
36
|
-
def controlled_vocab_text(field_name)
|
|
37
|
-
vocab_terms = @service.field_analyzer.controlled_vocab_terms
|
|
38
|
-
# based_near 'location' is handled specially because its controlled vocabulary is implemented differently
|
|
39
|
-
return unless vocab_terms.include?(field_name) || field_name == 'based_near'
|
|
40
|
-
'This property uses a controlled vocabulary.'
|
|
41
|
-
end
|
|
42
|
-
|
|
43
|
-
def split_text(mapping_key, controlled_vocab_info)
|
|
44
|
-
# regardless of schema, most controlled vocab fields only accept single values due to form limitations
|
|
45
|
-
return nil if controlled_vocab_info.present? && !mapping_key.in?(%w[location resource_type])
|
|
46
|
-
split_value = @service.mapping_manager.split_value_for(mapping_key)
|
|
47
|
-
return nil unless split_value
|
|
48
|
-
@split_formatter.format(split_value)
|
|
49
|
-
end
|
|
50
|
-
end
|
|
51
|
-
end
|
|
@@ -1,54 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Bulkrax
|
|
4
|
-
# Analyzes model fields and schemas
|
|
5
|
-
class SampleCsvService::FieldAnalyzer
|
|
6
|
-
attr_reader :field_list
|
|
7
|
-
|
|
8
|
-
def initialize(mappings)
|
|
9
|
-
@mappings = mappings
|
|
10
|
-
@field_list = []
|
|
11
|
-
@schema = nil
|
|
12
|
-
end
|
|
13
|
-
|
|
14
|
-
def find_or_create_field_list_for(model_name:)
|
|
15
|
-
existing = @field_list.find { |entry| entry.key?(model_name) }
|
|
16
|
-
return existing if existing.present?
|
|
17
|
-
|
|
18
|
-
klass = SampleCsvService::ModelLoader.determine_klass_for(model_name)
|
|
19
|
-
return {} if klass.nil?
|
|
20
|
-
|
|
21
|
-
model_entry = build_field_list_entry(model_name, klass)
|
|
22
|
-
@field_list << model_entry
|
|
23
|
-
model_entry
|
|
24
|
-
end
|
|
25
|
-
|
|
26
|
-
def controlled_vocab_terms
|
|
27
|
-
@field_list.flat_map do |hash|
|
|
28
|
-
hash.values.flat_map { |data| data["controlled_vocab_terms"] || [] }
|
|
29
|
-
end.uniq
|
|
30
|
-
end
|
|
31
|
-
|
|
32
|
-
private
|
|
33
|
-
|
|
34
|
-
def build_field_list_entry(model_name, klass)
|
|
35
|
-
schema_analyzer = SampleCsvService::SchemaAnalyzer.new(klass)
|
|
36
|
-
|
|
37
|
-
{
|
|
38
|
-
model_name => {
|
|
39
|
-
'properties' => extract_properties(klass),
|
|
40
|
-
'required_terms' => schema_analyzer.required_terms,
|
|
41
|
-
'controlled_vocab_terms' => schema_analyzer.controlled_vocab_terms
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
end
|
|
45
|
-
|
|
46
|
-
def extract_properties(klass)
|
|
47
|
-
if klass.respond_to?(:schema)
|
|
48
|
-
Bulkrax::ValkyrieObjectFactory.schema_properties(klass).map(&:to_s)
|
|
49
|
-
else
|
|
50
|
-
klass.properties.keys.map(&:to_s)
|
|
51
|
-
end
|
|
52
|
-
end
|
|
53
|
-
end
|
|
54
|
-
end
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Bulkrax
|
|
4
|
-
# Utility classes
|
|
5
|
-
class SampleCsvService::FilePathGenerator
|
|
6
|
-
def self.default_path
|
|
7
|
-
path = Rails.root.join('tmp', 'imports', "bulkrax_template_#{timestamp}.csv")
|
|
8
|
-
FileUtils.mkdir_p(path.dirname.to_s)
|
|
9
|
-
path
|
|
10
|
-
end
|
|
11
|
-
|
|
12
|
-
def self.timestamp
|
|
13
|
-
Time.current.utc.strftime('%Y%m%d_%H%M%S')
|
|
14
|
-
end
|
|
15
|
-
end
|
|
16
|
-
end
|
|
@@ -1,36 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Bulkrax
|
|
4
|
-
# Handles loading and filtering of Bulkrax field mappings
|
|
5
|
-
class SampleCsvService::MappingManager
|
|
6
|
-
attr_reader :mappings
|
|
7
|
-
|
|
8
|
-
def initialize
|
|
9
|
-
@mappings = load_mappings
|
|
10
|
-
end
|
|
11
|
-
|
|
12
|
-
def mapped_to_key(column_str)
|
|
13
|
-
@mappings.find { |_k, v| v["from"].include?(column_str) }&.first || column_str
|
|
14
|
-
end
|
|
15
|
-
|
|
16
|
-
def key_to_mapped_column(key)
|
|
17
|
-
@mappings.dig(key, "from")&.first || key
|
|
18
|
-
end
|
|
19
|
-
|
|
20
|
-
def find_by_flag(field_name, default)
|
|
21
|
-
@mappings.find { |_k, v| v[field_name] == true }&.first || default
|
|
22
|
-
end
|
|
23
|
-
|
|
24
|
-
def split_value_for(mapping_key)
|
|
25
|
-
@mappings.dig(mapping_key, "split")
|
|
26
|
-
end
|
|
27
|
-
|
|
28
|
-
private
|
|
29
|
-
|
|
30
|
-
def load_mappings
|
|
31
|
-
Bulkrax.field_mappings["Bulkrax::CsvParser"].reject do |_key, value|
|
|
32
|
-
value["generated"] == true
|
|
33
|
-
end
|
|
34
|
-
end
|
|
35
|
-
end
|
|
36
|
-
end
|
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Bulkrax
|
|
4
|
-
# Handles model loading based on configuration
|
|
5
|
-
class SampleCsvService::ModelLoader
|
|
6
|
-
attr_reader :models
|
|
7
|
-
|
|
8
|
-
def initialize(model_name)
|
|
9
|
-
@models = load_models(model_name)
|
|
10
|
-
end
|
|
11
|
-
|
|
12
|
-
def self.determine_klass_for(model_name)
|
|
13
|
-
if Bulkrax.config.object_factory == Bulkrax::ValkyrieObjectFactory
|
|
14
|
-
Valkyrie.config.resource_class_resolver.call(model_name)
|
|
15
|
-
else
|
|
16
|
-
model_name.constantize
|
|
17
|
-
end
|
|
18
|
-
rescue StandardError
|
|
19
|
-
nil
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
private
|
|
23
|
-
|
|
24
|
-
def load_models(model_name)
|
|
25
|
-
case model_name
|
|
26
|
-
when nil then []
|
|
27
|
-
when 'all' then all_available_models
|
|
28
|
-
else
|
|
29
|
-
model_name.constantize ? [model_name] : []
|
|
30
|
-
end
|
|
31
|
-
rescue StandardError
|
|
32
|
-
[]
|
|
33
|
-
end
|
|
34
|
-
|
|
35
|
-
def all_available_models
|
|
36
|
-
Hyrax.config.curation_concerns.map(&:name) +
|
|
37
|
-
[Bulkrax.collection_model_class&.name, Bulkrax.file_model_class&.name].compact
|
|
38
|
-
end
|
|
39
|
-
end
|
|
40
|
-
end
|
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Bulkrax
|
|
4
|
-
# Builds CSV rows (explanations and model data)
|
|
5
|
-
class SampleCsvService::RowBuilder
|
|
6
|
-
def initialize(service)
|
|
7
|
-
@service = service
|
|
8
|
-
@explanation_builder = SampleCsvService::ExplanationBuilder.new(service)
|
|
9
|
-
@value_determiner = SampleCsvService::ValueDeterminer.new(service)
|
|
10
|
-
end
|
|
11
|
-
|
|
12
|
-
def build_explanation_row(header_row)
|
|
13
|
-
@explanation_builder.build_explanations(header_row).map { |prop| prop.values.join(" ") }
|
|
14
|
-
end
|
|
15
|
-
|
|
16
|
-
def build_model_rows(header_row)
|
|
17
|
-
@service.all_models.map { |m| model_breakdown(m, header_row) }
|
|
18
|
-
end
|
|
19
|
-
|
|
20
|
-
private
|
|
21
|
-
|
|
22
|
-
def model_breakdown(model_name, header_row)
|
|
23
|
-
klass = SampleCsvService::ModelLoader.determine_klass_for(model_name)
|
|
24
|
-
return [] if klass.nil?
|
|
25
|
-
|
|
26
|
-
field_list = @service.field_analyzer.find_or_create_field_list_for(model_name: model_name)
|
|
27
|
-
|
|
28
|
-
header_row.map do |column|
|
|
29
|
-
@value_determiner.determine_value(column, model_name, field_list)
|
|
30
|
-
end
|
|
31
|
-
end
|
|
32
|
-
end
|
|
33
|
-
end
|