bulkrax 9.3.4 → 9.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8fc173a9153f3d6bf93ca309fda4a194cf6148eb26dc6b1bd9d4ec8e36e92cb0
-  data.tar.gz: d180567668e87809880fd7b86a3494ea2676c7afe7749d34d9da52b0728eed3b
+  metadata.gz: d7205ae1ec1e6287a3afe95e94df9e3f7afcdb72ad0d6e934cca44004b28e8a4
+  data.tar.gz: bae60650dc0ad54df42c9aa7da87829039462f6159c25b296a029c28f99006e8
 SHA512:
-  metadata.gz: 0f50fc0ac8d65266be8781a01f3bffeebf06177dc9253255ff0db130c3612b3e0d6e1cb6437cbb2445416a4e49dc31e8867c73ab423e7d34ae9a4a34b8b55c1b
-  data.tar.gz: 5b5162665af58619b3017297155ef07263bc1ccd5caf172e778f9f66eeb47545ea9cfdf84746786ef911433f3b29cc879997a67f64dd23df2f9e9a3b44658025
+  metadata.gz: 7b81763a20e21c294bf0fa56479cafc506fdc589f31c11d3bbd0383b28fda247734f3abe4d0797e5601abd64815aa457ea8d908bfbf427a03fa5e500bbfe60d2
+  data.tar.gz: e34566bafe813523345aaa41b2ad9fc2921e78be118250dfa911453fba185ea25cea6f74440fa6d5ddbb1dbcc699aca1f4ca5d3eb139c3dae65d27953de5f45d

data/app/assets/stylesheets/bulkrax/import_export.scss

@@ -1,6 +1,5 @@
 .bulkrax-card-footer {
-    min-height: 60px;
-
+  min-height: 60px;
 }
 
 div.importer_parser_fields_file_style span.radio {
@@ -45,3 +44,11 @@ div#s2id_exporter_export_source_collection {
   margin-left: 10px;
   margin-right: 10px;
 }
+
+// Make form input labels bold
+.simple_form label.control-label,
+.simple_form label.form-label,
+.simple_form .form-group > label,
+.simple_form .input > label {
+  font-weight: bold;
+}

data/app/controllers/bulkrax/importers_controller.rb

@@ -65,6 +65,15 @@ module Bulkrax
       end
     end
 
+    # POST /importers/sample_csv_file
+    def sample_csv_file
+      sample = Bulkrax::SampleCsvService.call(model_name: 'all', output: 'file')
+      send_file sample, filename: File.basename(sample), type: 'text/csv'
+    rescue StandardError => e
+      flash[:error] = "Unable to generate sample CSV file: #{e.message}"
+      redirect_back fallback_location: bulkrax.importers_path
+    end
+
     # GET /importers/1/edit
     def edit
       if api_request?

data/app/jobs/bulkrax/export_work_job.rb

@@ -18,12 +18,10 @@ module Bulkrax
       else
         if entry.failed?
           ExporterRun.increment_counter(:failed_records, args[1])
-          ExporterRun.decrement_counter(:enqueued_records, args[1]) unless exporter_run.reload.enqueued_records <= 0
-          raise entry.reload.current_status.error_class.constantize
         else
           ExporterRun.increment_counter(:processed_records, args[1])
-          ExporterRun.decrement_counter(:enqueued_records, args[1]) unless exporter_run.reload.enqueued_records <= 0
         end
+        ExporterRun.decrement_counter(:enqueued_records, args[1]) unless exporter_run.reload.enqueued_records <= 0
         # rubocop:enable Rails/SkipsModelValidations
       end
       return entry if exporter_run.reload.enqueued_records.positive?

data/app/services/bulkrax/sample_csv_service/column_builder.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  # Builds column headers for CSV
+  class SampleCsvService::ColumnBuilder
+    def initialize(service)
+      @service = service
+      @descriptor = SampleCsvService::ColumnDescriptor.new
+    end
+
+    def all_columns
+      required_columns + property_columns
+    end
+
+    def required_columns
+      mapped_core_columns +
+        relationship_columns +
+        file_columns
+    end
+
+    private
+
+    def mapped_core_columns
+      @descriptor.core_columns.map do |column|
+        @service.mapping_manager.key_to_mapped_column(column)
+      end
+    end
+
+    def property_columns
+      field_lists = @service.all_models.map do |m|
+        @service.field_analyzer.find_or_create_field_list_for(model_name: m)
+      end
+
+      properties = field_lists
+                   .flat_map { |item| item.values.flat_map { |config| config["properties"] || [] } }
+                   .uniq
+                   .map { |property| @service.mapping_manager.key_to_mapped_column(property) }
+                   .uniq
+
+      (properties - required_columns).sort
+    end
+
+    def relationship_columns
+      [
+        @service.mapping_manager.find_by_flag("related_children_field_mapping", 'children'),
+        @service.mapping_manager.find_by_flag("related_parents_field_mapping", 'parents')
+      ]
+    end
+
+    def file_columns
+      SampleCsvService::ColumnDescriptor::COLUMN_DESCRIPTIONS[:files].flat_map do |property_hash|
+        property_hash.keys.map do |key|
+          @service.mapping_manager.key_to_mapped_column(key)
+        end
+      end
+    end
+  end
+end

data/app/services/bulkrax/sample_csv_service/column_descriptor.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  # Manages column descriptions and metadata
+  class SampleCsvService::ColumnDescriptor
+    COLUMN_DESCRIPTIONS = {
+      include_first: [
+        { "model" => "The work types configured in your repository are listed below.\nIf left blank, your default work type, #{Bulkrax.default_work_type}, is used." },
+        { "source_identifier" => "This must be a unique identifier.\nIt can be alphanumeric with some special characters (e.g. hyphens, colons), and URLs are also supported." },
+        { "id" => "This column would optionally be included only if it is a re-import, i.e. for updating or deleting records.\nThis is a key identifier used by the system, which you wouldn't have for new imports." },
+        { "rights_statement" => "Rights statement URI for the work.\nIf not included, uses the value specified on the bulk import configuration screen." }
+      ],
+      visibility: [
+        { "visibility" => "Uses the value specified on the bulk import configuration screen if not added here.\nValid options: open, authenticated, restricted, embargo, lease" },
+        { "embargo_release_date" => "Required for embargo (yyyy-mm-dd)" },
+        { "visibility_during_embargo" => "Required for embargo" },
+        { "visibility_after_embargo" => "Required for embargo" },
+        { "lease_expiration_date" => "Required for lease (yyyy-mm-dd)" },
+        { "visibility_during_lease" => "Required for lease" },
+        { "visibility_after_lease" => "Required for lease" }
+      ],
+      files: [
+        { "file" => "Use filenames exactly matching those in your files folder.\nZip your CSV and files folder together and attach this to your importer." },
+        { "remote_files" => "Use the URLs to remote files to be attached to the work." }
+      ],
+      relationships: [
+        { "parents" => "The source_identifier or id of work or collection to be attached as parent." },
+        { "children" => "The source_identifier or id of work or file to be attached as child." }
+      ],
+      other: [
+        { "hide_from_catalog_search" => "Set to 1 to hide the collection from catalog search results." },
+        { "show_pdf_download_button" => "Set to 1 to show a PDF download link on the work's page." },
+        { "show_pdf_viewer" => "Set to 1 to show a PDF viewer on the work's page." },
+        { "video_embed" => "A valid URL to a hosted video that can appear in an iframe, beginning with 'http://' or 'https://'." }
+      ]
+    }.freeze
+
+    def core_columns
+      extract_column_names(:include_first) + extract_column_names(:visibility)
+    end
+
+    def find_description_for(column)
+      COLUMN_DESCRIPTIONS.each_value do |group|
+        prop = group.find { |hash| hash.key?(column) }
+        return prop[column] if prop
+      end
+      nil
+    end
+
+    private
+
+    def extract_column_names(group)
+      COLUMN_DESCRIPTIONS[group].map { |hash| hash.keys.first }
+    end
+  end
+end

data/app/services/bulkrax/sample_csv_service/csv_builder.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  # Builds CSV content
+  class SampleCsvService::CsvBuilder
+    IGNORED_PROPERTIES = %w[
+      admin_set_id alternate_ids
+      bulkrax_identifier
+      collection_type_gid contexts created_at
+      date date_modified date_uploaded depositor
+      embargo embargo_id
+      file_ids
+      has_model head
+      internal_resource is_child
+      lease lease_id
+      member_ids member_of_collection_ids modified_date
+      new_record
+      on_behalf_of owner proxy_depositor
+      rendering_ids representative_id
+      schema_version split_from_pdf_id state tail
+      thumbnail_id
+      updated_at
+    ].freeze
+
+    def initialize(service)
+      @service = service
+      @column_builder = SampleCsvService::ColumnBuilder.new(service)
+      @row_builder = SampleCsvService::RowBuilder.new(service)
+      @header_row = nil
+      @required_headings = []
+    end
+
+    def write_to_file(file_path)
+      FileUtils.mkdir_p(File.dirname(file_path))
+      CSV.open(file_path, "w") { |csv| write_rows(csv) }
+    end
+
+    def generate_string
+      CSV.generate { |csv| write_rows(csv) }
+    end
+
+    private
+
+    def write_rows(csv)
+      csv_rows.each { |row| csv << row }
+    end
+
+    def csv_rows
+      @header_row = fill_header_row
+      rows = [
+        @header_row,
+        @row_builder.build_explanation_row(@header_row),
+        *@row_builder.build_model_rows(@header_row)
+      ]
+      remove_empty_columns(rows)
+    end
+
+    def fill_header_row
+      @required_headings = @column_builder.required_columns
+      all_columns = @column_builder.all_columns
+      filtered = all_columns - IGNORED_PROPERTIES
+      @required_headings = @column_builder.required_columns & filtered
+      filtered
+    end
+
+    def remove_empty_columns(rows)
+      return rows if rows.empty?
+
+      columns = rows.transpose
+      non_empty_columns = columns.select { |col| keep_column?(col) }
+      non_empty_columns.transpose
+    end
+
+    def keep_column?(column)
+      heading = column[0]
+      return true if @required_headings.include?(heading)
+
+      # Check if any data row has content
+      column[2..-1].any? { |value| !value.nil? && value != "" && value != "---" }
+    end
+  end
+end

data/app/services/bulkrax/sample_csv_service/explanation_builder.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  # Builds explanations for CSV columns
+  class SampleCsvService::ExplanationBuilder
+    def initialize(service)
+      @service = service
+      @descriptor = SampleCsvService::ColumnDescriptor.new
+      @split_formatter = SampleCsvService::SplitFormatter.new
+    end
+
+    def build_explanations(header_row)
+      header_row.map do |column|
+        { column => build_explanation(column) }
+      end
+    end
+
+    private
+
+    def build_explanation(column)
+      mapping_key = @service.mapping_manager.mapped_to_key(column)
+
+      column_description = @descriptor.find_description_for(column)
+      controlled_vocab_info = controlled_vocab_text(mapping_key)
+      split_info = split_text(mapping_key, controlled_vocab_info)
+
+      components = [
+        column_description,
+        controlled_vocab_info,
+        split_info
+      ].compact
+
+      components.join("\n")
+    end
+
+    def controlled_vocab_text(field_name)
+      vocab_terms = @service.field_analyzer.controlled_vocab_terms
+      # based_near 'location' is handled specially because its controlled vocabulary is implemented differently
+      return unless vocab_terms.include?(field_name) || field_name == 'based_near'
+      'This property uses a controlled vocabulary.'
+    end
+
+    def split_text(mapping_key, controlled_vocab_info)
+      # regardless of schema, most controlled vocab fields only accept single values due to form limitations
+      return nil if controlled_vocab_info.present? && !mapping_key.in?(%w[location resource_type])
+      split_value = @service.mapping_manager.split_value_for(mapping_key)
+      return nil unless split_value
+      @split_formatter.format(split_value)
+    end
+  end
+end

data/app/services/bulkrax/sample_csv_service/field_analyzer.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  # Analyzes model fields and schemas
+  class SampleCsvService::FieldAnalyzer
+    attr_reader :field_list
+
+    def initialize(mappings)
+      @mappings = mappings
+      @field_list = []
+      @schema = nil
+    end
+
+    def find_or_create_field_list_for(model_name:)
+      existing = @field_list.find { |entry| entry.key?(model_name) }
+      return existing if existing.present?
+
+      klass = SampleCsvService::ModelLoader.determine_klass_for(model_name)
+      return {} if klass.nil?
+
+      model_entry = build_field_list_entry(model_name, klass)
+      @field_list << model_entry
+      model_entry
+    end
+
+    def controlled_vocab_terms
+      @field_list.flat_map do |hash|
+        hash.values.flat_map { |data| data["controlled_vocab_terms"] || [] }
+      end.uniq
+    end
+
+    private
+
+    def build_field_list_entry(model_name, klass)
+      schema_analyzer = SampleCsvService::SchemaAnalyzer.new(klass)
+
+      {
+        model_name => {
+          'properties' => extract_properties(klass),
+          'required_terms' => schema_analyzer.required_terms,
+          'controlled_vocab_terms' => schema_analyzer.controlled_vocab_terms
+        }
+      }
+    end
+
+    def extract_properties(klass)
+      if klass.respond_to?(:schema)
+        Bulkrax::ValkyrieObjectFactory.schema_properties(klass).map(&:to_s)
+      else
+        klass.properties.keys.map(&:to_s)
+      end
+    end
+  end
+end

data/app/services/bulkrax/sample_csv_service/file_path_generator.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  # Utility classes
+  class SampleCsvService::FilePathGenerator
+    def self.default_path
+      path = Rails.root.join('tmp', 'imports', "bulkrax_template_#{timestamp}.csv")
+      FileUtils.mkdir_p(path.dirname.to_s)
+      path
+    end
+
+    def self.timestamp
+      Time.current.utc.strftime('%Y%m%d_%H%M%S')
+    end
+  end
+end

data/app/services/bulkrax/sample_csv_service/mapping_manager.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  # Handles loading and filtering of Bulkrax field mappings
+  class SampleCsvService::MappingManager
+    attr_reader :mappings
+
+    def initialize
+      @mappings = load_mappings
+    end
+
+    def mapped_to_key(column_str)
+      @mappings.find { |_k, v| v["from"].include?(column_str) }&.first || column_str
+    end
+
+    def key_to_mapped_column(key)
+      @mappings.dig(key, "from")&.first || key
+    end
+
+    def find_by_flag(field_name, default)
+      @mappings.find { |_k, v| v[field_name] == true }&.first || default
+    end
+
+    def split_value_for(mapping_key)
+      @mappings.dig(mapping_key, "split")
+    end
+
+    private
+
+    def load_mappings
+      Bulkrax.field_mappings["Bulkrax::CsvParser"].reject do |_key, value|
+        value["generated"] == true
+      end
+    end
+  end
+end

data/app/services/bulkrax/sample_csv_service/model_loader.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  # Handles model loading based on configuration
+  class SampleCsvService::ModelLoader
+    attr_reader :models
+
+    def initialize(model_name)
+      @models = load_models(model_name)
+    end
+
+    def self.determine_klass_for(model_name)
+      if Bulkrax.config.object_factory == Bulkrax::ValkyrieObjectFactory
+        Valkyrie.config.resource_class_resolver.call(model_name)
+      else
+        model_name.constantize
+      end
+    rescue StandardError
+      nil
+    end
+
+    private
+
+    def load_models(model_name)
+      case model_name
+      when nil then []
+      when 'all' then all_available_models
+      else
+        model_name.constantize ? [model_name] : []
+      end
+    rescue StandardError
+      []
+    end
+
+    def all_available_models
+      Hyrax.config.curation_concerns.map(&:name) +
+        [Bulkrax.collection_model_class&.name, Bulkrax.file_model_class&.name].compact
+    end
+  end
+end

data/app/services/bulkrax/sample_csv_service/row_builder.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  # Builds CSV rows (explanations and model data)
+  class SampleCsvService::RowBuilder
+    def initialize(service)
+      @service = service
+      @explanation_builder = SampleCsvService::ExplanationBuilder.new(service)
+      @value_determiner = SampleCsvService::ValueDeterminer.new(service)
+    end
+
+    def build_explanation_row(header_row)
+      @explanation_builder.build_explanations(header_row).map { |prop| prop.values.join(" ") }
+    end
+
+    def build_model_rows(header_row)
+      @service.all_models.map { |m| model_breakdown(m, header_row) }
+    end
+
+    private
+
+    def model_breakdown(model_name, header_row)
+      klass = SampleCsvService::ModelLoader.determine_klass_for(model_name)
+      return [] if klass.nil?
+
+      field_list = @service.field_analyzer.find_or_create_field_list_for(model_name: model_name)
+
+      header_row.map do |column|
+        @value_determiner.determine_value(column, model_name, field_list)
+      end
+    end
+  end
+end

data/app/services/bulkrax/sample_csv_service/schema_analyzer.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  # Analyzes model schemas for required and controlled vocabulary fields
+  class SampleCsvService::SchemaAnalyzer
+    def initialize(klass)
+      @klass = klass
+      @schema = load_schema
+    end
+
+    def required_terms
+      return [] if @schema.blank?
+
+      @schema.select do |field|
+        field.respond_to?(:meta) &&
+          field.meta["form"].is_a?(Hash) &&
+          field.meta["form"]["required"] == true
+      end.map(&:name).map(&:to_s)
+    rescue StandardError
+      []
+    end
+
+    def controlled_vocab_terms
+      return [] unless @schema
+
+      controlled_properties = extract_controlled_properties
+      controlled_properties.empty? ? registered_controlled_vocab_fields : controlled_properties
+    rescue StandardError
+      []
+    end
+
+    private
+
+    def load_schema
+      return nil unless @klass.respond_to?(:schema)
+      # Yes, this looks strange. The fallback is intentional.
+      # At the point in time when this service is being created, Hyrax behaves
+      # differently between flexible metadata setting on & off. This may be modified
+      # in the future, and this code can be revisited then.
+      # flexible=true: @klass.new.singleton_class.schema would return the full schema,
+      #                but @klass.schema doesn't get the flexible metadata terms
+      # flexible=false: @klass.new.singleton_class.schema returns nil so it will fallback
+      @klass.new.singleton_class.schema || @klass.schema
+    rescue StandardError
+      nil
+    end
+
+    def extract_controlled_properties
+      return [] unless @schema
+
+      @schema.filter_map do |property|
+        next unless property.respond_to?(:meta)
+        sources = property.meta&.dig('controlled_values', 'sources')
+        next if sources.nil? || sources == ['null'] || sources == 'null'
+        property.name.to_s
+      end
+    end
+
+    def registered_controlled_vocab_fields
+      qa_registry.filter_map do |k, v|
+        k.singularize if v.klass == Qa::Authorities::Local::FileBasedAuthority
+      end
+    end
+
+    def qa_registry
+      @qa_registry ||= Qa::Authorities::Local.registry.instance_variable_get('@hash')
+    end
+  end
+end

data/app/services/bulkrax/sample_csv_service/split_formatter.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  # Formats split pattern descriptions
+  class SampleCsvService::SplitFormatter
+    def format(split_value)
+      return "Property does not split." if split_value.nil?
+
+      if split_value == true
+        parse_pattern(Bulkrax.multi_value_element_split_on.source)
+      elsif split_value.is_a?(String)
+        parse_pattern(split_value)
+      else
+        split_value
+      end
+    end
+
+    private
+
+    def parse_pattern(pattern)
+      chars = extract_characters(pattern)
+      format_message(chars)
+    end
+
+    def extract_characters(pattern)
+      if (match = pattern.match(/\[([^\]]+)\]/))
+        match[1]
+      elsif (single = pattern.match(/\\(.)/))
+        single[1]
+      else
+        pattern
+      end
+    end
+
+    def format_message(chars)
+      formatted = chars.chars.then do |c|
+        c.length > 1 ? "#{c[0..-2].join(' ')}, or #{c.last}" : c.first
+      end
+      "Split multiple values with #{formatted}"
+    end
+  end
+end

data/app/services/bulkrax/sample_csv_service/value_determiner.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  # Determines values for CSV cells
+  class SampleCsvService::ValueDeterminer
+    def initialize(service)
+      @service = service
+      @column_builder = SampleCsvService::ColumnBuilder.new(service)
+    end
+
+    def determine_value(column, model_name, field_list)
+      key = @service.mapping_manager.mapped_to_key(column)
+      required_terms = field_list.dig(model_name, 'required_terms')
+
+      if field_list.dig(model_name, "properties")&.include?(key)
+        mark_required_or_optional(key, required_terms)
+      elsif special_column?(column, key)
+        special_value(column, key, model_name, required_terms)
+      end
+    end
+
+    private
+
+    def special_column?(column, key)
+      descriptor = SampleCsvService::ColumnDescriptor.new
+      visibility_cols = descriptor.send(:extract_column_names, :visibility)
+
+      key.in?(['model', 'work_type']) ||
+        column.in?(visibility_cols) ||
+        column == 'source_identifier' ||
+        column == 'rights_statement' ||
+        relationship_column?(column) ||
+        file_column?(column)
+    end
+
+    def special_value(column, key, model_name, required_terms)
+      return SampleCsvService::ModelLoader.determine_klass_for(model_name).to_s if key.in?(['model', 'work_type'])
+      return 'Required' if column == 'source_identifier'
+      return mark_required_or_optional(key, required_terms) if column == 'rights_statement'
+      # collections do not have files
+      return nil if file_column?(column) && model_name.in?([Bulkrax.collection_model_class].compact.map(&:to_s))
+      'Optional'
+    end
+
+    def mark_required_or_optional(field, required_terms)
+      return 'Unknown' unless required_terms
+      required_terms.include?(field) ? 'Required' : 'Optional'
+    end
+
+    def relationship_column?(column)
+      relationships = [
+        @service.mapping_manager.find_by_flag("related_children_field_mapping", 'children'),
+        @service.mapping_manager.find_by_flag("related_parents_field_mapping", 'parents')
+      ]
+      column.in?(relationships)
+    end
+
+    def file_column?(column)
+      file_cols = SampleCsvService::ColumnDescriptor::COLUMN_DESCRIPTIONS[:files].flat_map do |property_hash|
+        property_hash.keys.filter_map do |key|
+          @service.mappings.dig(key, "from")&.first
+        end
+      end
+      column.in?(file_cols)
+    end
+  end
+end