cdm_migrator 3.3.2 → 3.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 88e952c96bf9c9088bfd1b1b6b5ccd0ff0ea28089440767f55e7232b20aeec69
-  data.tar.gz: 53ca89586e9f42f5318301846ac34ed7cca77ff5ed4b27e86d956746726eb1b5
+  metadata.gz: 9bd9baa9f5c277e22421a1726d4652d2231a4067aabcf2e4f048ac9aa7264fc2
+  data.tar.gz: f698973f8726c227326fd7e0c393af4201c7dd9352f43d669cff9659f310da85
 SHA512:
-  metadata.gz: 1e08e51b6c1a5dcd2d21c8968158e8c1ab6a12d68c46fcf16854c5b3019c622bd382726b85d570e9457f80c8093548972c982d3e9d4696ad02eff7448c8c590f
-  data.tar.gz: 9d6e141f1abac1ebca916d51d7727e692ba0d18f7f4f6e9a91e206b1c293fd5d5fc64ad1ea154990913e9796ed627a5085bfebbceeaf0420690035fa16af3efa
+  metadata.gz: 6c264c4d7a122f598aef1d746766771b408defb9a15e014490f7a2acd43772197c5e66d15a99120c13089f013fe702f9dc5aeaebbb9cbc9f70b2110b54cac55d
+  data.tar.gz: 85241bcb0111ead83629678cfde68fb729d49997890dbc0432857e6f7976f74f3f4296b0e6aecac309b71a84b398ea02fa24f1d346d652912a60eded09c39c0c

data/app/controllers/cdm_migrator/csv_controller.rb

@@ -29,7 +29,7 @@ module CdmMigrator
 
     def upload
       @admin_sets  = AdminSet.all.map { |as| [as.title.first, as.id] }
-      @collections = Collection.all.map { |col| [col.title.first, col.id] }
+      @collections = Hyrax.config.collection_class.all.map { |col| [col.title.first, col.id] }
     end
 
     def create
@@ -88,7 +88,7 @@ module CdmMigrator
     end
 
     def edit
-      @collections = ::Collection.all.map { |c| [c.title.first, c.id] }
+      @collections = Hyrax.config.collection_class.all.map { |c| [c.title.first, c.id] }
     end
 
     def update
@@ -102,7 +102,7 @@ module CdmMigrator
         elsif type.include? "Work"
           metadata = create_data(row.except('id', 'type'), work_form(type), obj, mvs)
         elsif type.include? "File"
-          metadata = create_data(row.except('id', 'type'), work_form(type), obj, mvs)
+          metadata = create_data(row.except('id', 'type'), file_form, obj, mvs)
         end
         unless metadata.nil?
           obj.attributes = metadata
@@ -114,28 +114,19 @@ module CdmMigrator
     end
 
     def export
-      solr = RSolr.connect url: Account.find_by(tenant: Apartment::Tenant.current).solr_endpoint.url
+      # Get a collection's member works from Solr
+      solr = RSolr.connect url: Blacklight.connection_config[:url]
       response = solr.get 'select', params: {
           q: "member_of_collection_ids_ssim:#{params[:collection_id]}",
+          fq: ["has_model_ssim:FileSet OR has_model_ssim:*Work"],
           rows: 3400,
           fl: "id"
       }
       unless response['response']['docs'].empty? || response['response']['docs'][0].empty?
         work_ids = response['response']['docs'].map { |doc| doc['id'] }
       end
-      #works    = ::ActiveFedora::Base.where member_of_collection_ids_ssim: params[:collection_id]
-      @csv_headers = ['type'] + work_fields
-      @csv_array   = [@csv_headers.join(',')]
-      work_ids.each do |work_id|
-        doc = ::SolrDocument.find work_id
-        add_line doc
-        doc._source[:file_set_ids_ssim].each do |file_id|
-          file_doc = ::SolrDocument.find file_id
-          add_line file_doc
-        end
-      end
 
-      send_data @csv_array.join("\n"),
+      send_data CsvExportService.new(available_works).csv_for(work_ids),
                 :type => 'text/csv; charset=iso-8859-5; header=present',
                 :disposition => "attachment; filename=export.csv"
     end
@@ -146,43 +137,6 @@ module CdmMigrator
       authorize! :create, available_works.first
     end
 
-    def add_line doc
-      line_hash = {}
-      line_hash['type'] = doc._source[:has_model_ssim].first
-      work_fields.each do |field|
-        line_hash[field] = create_cell doc, field
-      end
-      @csv_array << line_hash.values_at(*@csv_headers).map { |cell| cell = '' if cell.nil?; "\"#{cell.gsub("\"", "\"\"")}\"" }.join(',')
-
-    end
-
-    def work_fields
-      @fields ||=  available_works.map { |work| work.new.attributes.keys }.flatten.uniq - excluded_fields
-    end
-
-    def excluded_fields
-      %w[date_uploaded date_modified head tail state proxy_depositor on_behalf_of arkivo_checksum label
-       relative_path import_url part_of resource_type access_control_id
-       representative_id thumbnail_id rendering_ids admin_set_id embargo_id
-       lease_id]
-    end
-
-    def create_cell w, field
-      if field.include? 'date' or field == 'chronological_coverage'
-        if w._source[field+'_tesim'].is_a?(Array)
-          w._source[field+'_tesim'].join('|')
-        else
-          w._source[field+'_tesim']
-        end
-      elsif w.respond_to?(field.to_sym)
-        if w.send(field).is_a?(Array)
-          w.send(field).join('|')
-        else
-          w.send(field)
-        end
-      end
-    end
-
     def available_works
       @available_works ||= Hyrax::QuickClassificationQuery.new(current_user).authorized_models
     end
@@ -277,7 +231,7 @@ module CdmMigrator
       if file_path.present? && File.file?(file_path) && @max_file_size
         if File.size(file_path.gsub("file://", "")) > @max_file_size
           @error_list[row_number] = { "file size" => "The file at #{file_path} is too large to be uploaded. Please compress the file or split it into parts.
-                                                      Each part should be under #{helpers.number_to_human_size(@max_file_size)}." }
+                                                    Each part should be under #{helpers.number_to_human_size(@max_file_size)}." }
         end
       end
     end
@@ -346,9 +300,9 @@ module CdmMigrator
                 hash[field.to_s] = "May contain the wrong multi-value separator or a typo in the URI."
               end
             else # Or val should be string
-              invalid_chars = ["\\"]
-              # Make exceptions for backslashes that are part of whitespace characters
-              # by deleting them before checking for stray \s
+            invalid_chars = ["\\"]
+               # Make exceptions for backslashes that are part of whitespace characters
+               # by deleting them before checking for stray \s
               if val.delete("\t\r\n\s\n").match Regexp.union(invalid_chars)
                 hash[field.to_s] = "May contain an invalid character such as #{invalid_chars.to_sentence(last_word_connector: ", or ")}."
               end

data/app/jobs/cdm_migrator/batch_create_files_with_ordered_members_job.rb

@@ -14,32 +14,59 @@ module CdmMigrator
     #     advantage of the same strategy as Hyrax::Actors::FileSetOrderedMembersActor
     #     but you don't need the OrderedMembersActor constant initialized.
 
-    # This rescue is a safeguard against creating lots of orphan file sets if there
-    # are recurring errors (see https://tinyurl.com/nh4c5e9j). Instead, CdmMigrator
-    # will fall back to creating file sets one-by-one if this job fails once.
-    rescue_from(StandardError) do |exception|
-      Rails.logger.error "BatchCreateFilesWithOrderedMembersJob error: #{exception.to_s}"
-      RestartUploadFromMiddleJob.perform_later(arguments[0], arguments[1], arguments[2])
+    def perform work, ingest_work, user
+      # Reload the work to get the most recent and accurate member associations
+      work.reload
+      if work.ordered_members.to_a.empty? && work.file_sets.empty?
+        attach_files(work, ingest_work.files, user)
+      else
+        delete_excess_file_sets(work)
+        ordered_count = work.reload.ordered_members.to_a.count
+        unless ordered_count == ingest_work.files.count
+          # Attach any files that might be missing
+          files = ingest_work[ordered_count..]
+          attach_files(work, files, user)
+        end
+      end
+      first_file_set = work.ordered_members.to_a.first
+      work.representative = first_file_set
+      work.thumbnail = first_file_set
+      work.save!
+      work.file_sets.each { |fs| CdmIngestFilesJob.perform_later(fs, fs.import_url, user, ingest_work) }
     end
 
-    def perform work, ingest_work, user
-        ordered_members = []
-        ingest_work.files.each do |file|
-          url = file[:url]
-          last_file = ingest_work.files.last==file
-          ::FileSet.new(import_url: url, label: file[:title]) do |fs|
-            fs.attributes = file[:metadata]
-            fs.save!
-            ordered_members << fs
-          end
+    private
+
+    def attach_files(work, ingest_work_files, user)
+      ingest_work_files.each do |file|
+        url = file[:url]
+        ordered_members = work.ordered_members
+        # last_file = ingest_work.files.last==file
+        ::FileSet.new(import_url: url, label: file[:title]) do |fs|
+          fs.attributes = file[:metadata]
+          fs.save!
+          ordered_members << fs
         end
-        actor = Hyrax::Actors::OrderedMembersActor.new(ordered_members, user)
-        actor.attach_ordered_members_to_work(work)
-        work.representative = work.ordered_members.to_a.first
-        work.thumbnail_id = work.ordered_member_ids.first
+      end
+      work.save!
+      work.reload.ordered_members.to_a.each do |file_set|
+        Hyrax.config.callback.run(:after_create_fileset, file_set, user, warn: false)
+      end
+    end
+
+    # Sometimes when this job fails, file sets are attached to the work
+    # without attaching them as ordered members. This creates "ghost files"
+    # that don't show up in the interface but are still linked to the work as members
+    def delete_excess_file_sets(work)
+      ordered_members = work.ordered_members.to_a
+      ghost_members = work.file_sets.select { |fs| ordered_members.exclude? fs }
+      if ghost_members.any?
+        # Unlink the file sets from the parent work first because it makes deleting them faster
+        work.members = ordered_members
         work.save!
-        work.file_sets.each { |fs| CdmIngestFilesJob.perform_later(fs, fs.import_url, user, ingest_work) }
+        ghost_members.each(&:destroy!)
+      end
     end
-    
+
   end
 end

data/app/jobs/cdm_migrator/create_work_job.rb

@@ -4,20 +4,15 @@ module CdmMigrator
 
     def perform(ingest_work, user, admin_set_id, collection_id)
       admin_set = ::AdminSet.find(admin_set_id) rescue nil
-      collection = ::Collection.find(collection_id) rescue nil
+      collection = Hyrax.config.collection_class.find(collection_id) rescue nil
       work = Object.const_get(ingest_work.work_type).new
-      #status_after, embargo_date, lease_date = nil, nil, nil
       work.apply_depositor_metadata(user)
       work.attributes = ingest_work.data
-      if ingest_work.data.has_key? 'downloadable'
-        # Convert string to boolean
-        work.downloadable = ActiveModel::Type::Boolean.new.cast(ingest_work.data['downloadable'])
-      elsif work.attributes.include? 'downloadable' # Set work to downloadable by default
-        work.downloadable = true
-      end
       work.member_of_collections = [collection] if collection
       work.admin_set = admin_set if admin_set
       work.date_uploaded = DateTime.now
+      add_configured_permissions(work)
+      work.try(:to_controlled_vocab)
       begin
         work.save!
       # Weird error where descriptions with whitespace chars \n or \r don't save the 1st time
@@ -29,9 +24,39 @@ module CdmMigrator
         work.description = old_descr
         work.save!
       end
-      # To use the BatchCreateFilesWithOrderedMembersJob instead, replace the following line
-      # with BatchCreateFilesWithOrderedMembersJob.perform_later(work, ingest_work, user)
-      BatchCreateFilesJob.perform_later(work, ingest_work, user)
+      # Creating file (sets) with Hyrax::Actors::OrderedMembersActor is now the default.
+      # To use the original Hyrax::Actors::FileSetActor, replace the line below with
+      # BatchCreateFilesJob.perform_later(work, ingest_work, user)
+      BatchCreateFilesWithOrderedMembersJob.perform_later(work, ingest_work, user)
     end
+
+    private
+
+    def add_configured_permissions(work)
+      work_type = work.class
+      return if configured_permissions.nil? # Nothing configured at all
+
+      permissions_config = configured_permissions.dig(work_type.to_s)
+
+      return if permissions_config.nil? # Nothing configured for this work type
+
+      permissions = permissions_config.map do |permission_level, group_name|
+        # Check if the permission level is configured in Hyrax
+        raise "#{permission_level} permission is not configured in this repo. Is it set in Hyrax.config.permission_levels?" unless Hyrax.config.permission_levels.index(permission_level).presence
+        # Construct each permission as a hash
+        group_name.map do |group|
+          { name: group, type: "group", access: permission_level }
+        end.flatten
+      end.flatten
+
+      # Finally, set the work's permissions_attributes
+      work.permissions_attributes = permissions
+    end
+
+    # @return[Hash] - A hash like { "GenericWork"=> { "edit" => ["admin"], "download" => ["public"] } }
+    def configured_permissions
+      CdmMigrator::Engine.config.dig('default_work_permissions')
+    end
+
   end
 end

data/app/services/csv_export_service.rb

@@ -0,0 +1,84 @@
+module CdmMigrator
+  class CsvExportService
+
+    # A service for exporting work and file set metadata to a csv. You can customize headers/fields by
+    # overriding the included_fields method below. You can also specify different work types by changing/overriding
+    # the available_work_types method.
+
+    # @param [Array <Class>] - the available work types, passed in from the controller, such as GenericWork
+    def initialize(work_types)
+      @work_types = work_types
+    end
+
+    # @param [Array <String>] - the work ids (for GenericWork or other work type) to export metadata for
+    # @param [String] - where to save the csv file to (filepath)
+    def write_to_csv(work_ids, filepath)
+      File.open(filepath, 'w') { |file| file.write(rows_for(work_ids).map(&:to_csv).join) }
+    end
+
+    def csv_for(work_ids)
+      rows_for(work_ids).map(&:to_csv).join
+    end
+
+    # @param [Array <String>] - the work ids (for GenericWork or other work type) to export metadata for
+    # @return [Array <Array>] - An array of arrays where each nested array contains the metadata
+    # for a work or file set and corresponds to a csv row.
+    def rows_for(work_ids)
+      csv_array = [csv_headers]
+      work_ids.each_with_object(csv_array).each do |work_id|
+        doc = ::SolrDocument.find work_id
+        csv_array << row_for(doc)
+        doc._source[:file_set_ids_ssim].each do |file_id|
+          file_doc = ::SolrDocument.find file_id
+          csv_array << row_for(file_doc)
+        end
+      end
+    end
+
+    # @param [SolrDocument] - Any model that has the properties listed in #included_fields (e.g. GenericWork, FileSet)
+    # @return [Array <String>] - the csv row for the given document
+    def row_for(document)
+      line_hash = {}
+      line_hash['type'] = document._source[:has_model_ssim].first
+      included_fields.each do |field|
+        line_hash[field] = create_cell document, field
+      end
+      line_hash.values_at(*csv_headers).map { |cell| cell.blank? ? '' : cell }
+    end
+
+    private
+
+    # @return [Array <String>]
+    def included_fields
+      @work_types.map { |work| work.new.attributes.keys }.flatten.uniq - excluded_fields
+    end
+
+    def excluded_fields
+      %w[date_uploaded date_modified head tail state proxy_depositor on_behalf_of arkivo_checksum label
+       relative_path import_url part_of resource_type access_control_id
+       representative_id thumbnail_id rendering_ids admin_set_id embargo_id
+       lease_id]
+    end
+
+    # @param [SolrDocument] - the document to create a cell for
+    # @param [String or Symbol] - the name of the field
+    # NOTE: any fields you want to include must also be added to the SolrDocument model as methods
+    # because of the check for respond_to?
+    def create_cell document, field
+      properties = document.hydra_model.properties
+      if document.respond_to?(field.to_sym)
+        if properties.keys.include?(field) && properties[field].multiple? && field.to_sym != :doi
+          document.send(field).join('|')
+        else
+          document.send(field)
+        end
+      end
+    end
+
+    # @return [Array <String>] - the heaaders for the csv
+    def csv_headers
+      ['type'] + included_fields
+    end
+
+  end
+end

data/lib/cdm_migrator/version.rb

@@ -1,3 +1,3 @@
 module CdmMigrator
-  VERSION = '3.3.2'
+  VERSION = '3.5.2'
 end

data/lib/generators/cdm_migrator/install/templates/config/cdm_migrator.yml

@@ -51,3 +51,7 @@ default_fields:
   - identifier
   - based_near
   - related_url
+default_work_permissions:
+#  GenericWork:     # the work type to set default permissions for
+#    edit:          # the permission level (must also be set in config/initializers/hyrax.rb: Hyrax.config.permission_levels = ...). Default available values are edit, read
+#      - admin      # the group name (currently only groups are supported, not user names)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cdm_migrator
 version: !ruby/object:Gem::Version
-  version: 3.3.2
+  version: 3.5.2
 platform: ruby
 authors:
 - sephirothkod
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-06-26 00:00:00.000000000 Z
+date: 2024-07-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -24,6 +24,26 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: hyrax
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: engine_cart
   requirement: !ruby/object:Gem::Requirement
@@ -98,6 +118,7 @@ files:
 - app/models/cdm_migrator/application_record.rb
 - app/models/cdm_migrator/batch_ingest.rb
 - app/models/cdm_migrator/ingest_work.rb
+- app/services/csv_export_service.rb
 - app/views/cdm_migrator/cdm/collection.html.erb
 - app/views/cdm_migrator/cdm/mappings.html.erb
 - app/views/cdm_migrator/csv/_batches_list.html.erb
@@ -141,7 +162,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.5.3
 signing_key: 
 specification_version: 4
 summary: ContentDM to Hyrax migrator.