cdm_migrator 3.0.1 → 3.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 985574c3d7f661fd0f9d1ce9a15e9269fa4c2dafd10d387110695400aff2b762
-  data.tar.gz: 0c24c2d7beb6f6d9c66a44dd30393c8dae0cb9222c9d6c69186909cdfe074590
+  metadata.gz: 8bc6e448326179791de466ba5dc1d35a26fd48fa2a85a8fbabeb7d8dad60339e
+  data.tar.gz: f3cf46dec46cd8166bc0555eef28e2868db6fbbf77963083f93fca1958b00e19
 SHA512:
-  metadata.gz: 6b06e22ca127a1b5fa304c18edb939c5082cb3251b7c5be202e51f6914e6ab703a3a7c2a3cc992327ae0122eacc52eec5d5ae810e00eb8531b5e49f5a9481a71
-  data.tar.gz: 2cefe7be69db9247165ec1c03eeadd5bbffbe0b1ed8044f097e6a6070968e77b224706845eb6072d891cdd5718612c93544321b7a2d9a204af8e92ff4cf6388d
+  metadata.gz: 9d8b8bc4d318269375c82429cbcd4bb2e3d07b1761ef54854b262746c09e46430ee3d9da02122567f5244599bb3f312b7248b1b0a610b12abc2a049b0b7470dc
+  data.tar.gz: 4dfc8a920f09e1e3c84b5eb09116adadcbf44556c3abe4b4daa1dbce7e87e767ea380de9fbdc5368c548032bf4b6f074a5ab0268fd6600dff75a9530445a7210

data/README.md

@@ -27,11 +27,12 @@ to insert the yml and add a link to your Hyrax dashboard
 
 ## Usage
 1. Add your ContentDM url and api port to the cdm_migrator.yml file.
-2. Navigate to the *cdm_migrator/collection* url to select your contentdm collection and what type of work you want to export it to and click "choose mappings".
-3. Map the ContentDM fields to your Hyrax work and file fields\* and click "generate CSV".
-4. Refine the CSV as you see fit.
-5. Navigate to the *cdm_migrator/upload* url; choose your multi-value seperator (default is |) and upload your CSV file.
-6. Done.
+2. Configure the CSV Checker with the appropriate fields, paths, or multi-value separator (in cdm_migrator.yml).
+3. Navigate to the *cdm_migrator/collection* url to select your contentdm collection and what type of work you want to export it to and click "choose mappings".
+4. Map the ContentDM fields to your Hyrax work and file fields\* and click "generate CSV".
+5. Refine the CSV as you see fit. Optional: go to *cdm_migrator/csv_checker* and upload it to validate metadata fields and/or file paths.
+6. Navigate to the *cdm_migrator/upload* url; choose your multi-value separator (default is |) and upload your CSV file.
+7. Done.
 
 \* cdm_migrator uses the generated Hyrax forms (ex. Hyrax::Forms::GenericWorkForm) in your host application to obtain it's terms for mapping. If you have added terms to your FileSet model extend the Hyrax::Forms::FileSetEditForm with Hyrax::FileSetForm in your host application so that the changes will be detected by the migrator. You can also add a list of fields in the yml file, under "default fields".
 

data/app/assets/stylesheets/cdm_migrator/csv_checker.css

@@ -0,0 +1,37 @@
+.csv-collapse-link.collapsed {
+    display: inline-block;
+    vertical-align: top;
+}
+
+#csv-collapse-link-text::after {
+    content: "❯";
+    display: inline-block;
+    font-size: smaller;
+    right: 15px;
+    transform: rotate(90deg);
+    margin-left: 0.5em;
+}
+
+.collapsed #csv-collapse-link-text::after {
+    transform: rotate(0deg);
+    transition: transform 0.1s ease;
+    margin-left: 2px;
+    vertical-align: top;
+}
+
+#errors-explanation ul li {
+    list-style-type: none;
+    margin-bottom: 2px;
+}
+
+#errors-explanation ul {
+    padding-left: 0;
+}
+
+#errors-explanation {
+    margin-top: 5px;
+}
+
+#csv-form {
+    margin-top: 1em;
+}

data/app/controllers/cdm_migrator/cdm_controller.rb

@@ -104,10 +104,10 @@ module CdmMigrator
 
 		def load_yaml
 			stripped_url = request.base_url.dup.gsub(/https?:\/\//, '').gsub(/:[0-9]*/,'')
-			if CdmMigrator::Engine.config['cdm_api'].key? stripped_url
-				tenant = CdmMigrator::Engine.config['cdm_api'][stripped_url]
+			if CdmMigrator::Engine.config['tenant_settings'].key? stripped_url
+				tenant = CdmMigrator::Engine.config['tenant_settings'][stripped_url]['cdm_api']
 			else
-				tenant = CdmMigrator::Engine.config['cdm_api']['default']
+				tenant = CdmMigrator::Engine.config['tenant_settings']['default']['cdm_api']
 			end
 			@cdm_url = tenant['url']
 			@cdm_port = tenant['port']
@@ -121,7 +121,7 @@ module CdmMigrator
 			filename = child ? child['pagefile'] : "#{rec.first}.#{rec.last}"
 
 			if params[:file_system]=='true'
-				"file://#{file_path(rec.first)}"
+				"file://#{file_path(cisoptr)}"
 			elsif @cdm_api == 'server'
 				"#{@cdm_url}:#{@cdm_port}/cgi-bin/showfile.exe?CISOROOT=#{params[:collection]}&CISOPTR=#{cisoptr}"
 			else

data/app/controllers/cdm_migrator/csv_controller.rb

@@ -1,20 +1,20 @@
 module CdmMigrator
-	class CsvController < ApplicationController
-		helper_method :default_page_title, :admin_host?, :available_translations, :available_works
+  class CsvController < ApplicationController
+    helper_method :default_page_title, :admin_host?, :available_translations, :available_works
     include ActionView::Helpers::UrlHelper
-		layout 'hyrax/dashboard' if Hyrax
+    layout 'hyrax/dashboard' if Hyrax
     before_action :authenticate, except: :index
+    before_action :load_config, only: :csv_checker
 
-    def file_path_checker
-     if params[:file]
-       check_paths params[:file].path
-
-       if @path_list.blank?
-         flash[:notice] = "All file paths are valid."
-       else
-         flash[:error] = "Cdm Migrator couldn't find files at the following urls. Please correct the paths and try again."
-       end
-     end
+    def csv_checker
+      if params[:file]
+        check_csv params[:file].path
+        if @error_list.blank?
+          flash[:notice] = "All data are valid."
+        else
+          flash[:error] = "The CSV Checker found some errors in the CSV. Please correct them and check again."
+        end
+      end
     end
 
     def index
@@ -27,34 +27,34 @@ module CdmMigrator
       end
     end
 
-		def upload
-			@admin_sets = AdminSet.all.map { |as| [as.title.first, as.id] }
-			@collections = Collection.all.map { |col| [col.title.first, col.id] }
-		end
+    def upload
+      @admin_sets  = AdminSet.all.map { |as| [as.title.first, as.id] }
+      @collections = Collection.all.map { |col| [col.title.first, col.id] }
+    end
 
-		def create
-			dir = Rails.root.join('public', 'uploads', 'csvs')
-			FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
-			time = DateTime.now.strftime('%s')
-			filename = params[:csv_import][:csv_file].original_filename.gsub('.csv',"#{time}.csv")
-      csv = dir.join(filename).to_s
-			File.open(csv, 'wb') do |file|
-			  file.write(params[:csv_import][:csv_file].read)
+    def create
+      dir = Rails.root.join('public', 'uploads', 'csvs')
+      FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
+      time     = DateTime.now.strftime('%s')
+      filename = params[:csv_import][:csv_file].original_filename.gsub('.csv', "#{time}.csv")
+      csv      = dir.join(filename).to_s
+      File.open(csv, 'wb') do |file|
+        file.write(params[:csv_import][:csv_file].read)
       end
-      check_paths csv
-      if @path_list.present?
-        flash[:error] = "some file paths are invalid"
+      check_csv csv
+      if @error_list.present?
+        flash[:error] = "Cdm Migrator found some problems with the CSV. Use the CSV Checker for more details."
       end
       parse_csv(csv, params[:csv_import][:mvs])
 
       ingest = BatchIngest.new({
-                                   data: @works,
-                                   size: @works.length,
-                                   csv: csv,
-                                   admin_set_id: params[:admin_set],
+                                   data:          @works,
+                                   size:          @works.length,
+                                   csv:           csv,
+                                   admin_set_id:  params[:admin_set],
                                    collection_id: params[:collection],
-                                   user_id: current_user.id,
-                                   message: @path_list.blank? ? nil : @path_list.to_s.gsub("\"","&quot;")
+                                   user_id:       current_user.id,
+                                   message:       @path_list.blank? ? nil : @path_list.to_s.gsub("\"", "&quot;")
                                })
       if ingest.save! && @path_list.blank?
         BatchCreateWorksJob.perform_later(ingest, current_user)
@@ -75,9 +75,8 @@ module CdmMigrator
     end
 
     def generate
-      headers = ['type','url']
-      skip = ["id", "head", "tail", "depositor", "date_uploaded", "date_modified", "import_url", "thumbnail_id",
-              "embargo_id", "lease_id", "access_control_id", "representative_id"]
+      headers = %w(type url)
+      skip    = %w(id head tail depositor date_uploaded date_modified import_url thumbnail_id embargo_id lease_id access_control_id representative_id)
       GenericWork.new.attributes.each do |key, val|
         headers << "work_#{key}" unless skip.include? key
       end
@@ -87,19 +86,109 @@ module CdmMigrator
       fname = "template_#{DateTime.now.to_i}"
       render plain: CSV.generate { |csv| csv << headers }, content_type: 'text/csv'
     end
-		
-		private
+
+    def edit
+      @collections = ::Collection.all.map { |c| [c.title.first, c.id] }
+    end
+
+    def update
+      mvs = params[:csv_update][:mvs]
+      csv = CSV.parse(params[:csv_update][:csv_file].read, headers: true, encoding: 'utf-8').map(&:to_hash)
+      csv.each do |row|
+        obj = ActiveFedora::Base.find row['id']
+        type = row.first.last
+        if type.nil?
+          next
+        elsif type.include? "Work"
+          metadata = create_data(row.except('id', 'type'), work_form(type), obj, mvs)
+        elsif type.include? "File"
+          metadata = create_data(row.except('id', 'type'), work_form(type), obj, mvs)
+        end
+        unless metadata.nil?
+          obj.attributes = metadata
+          obj.save
+        end
+      end
+      flash[:notice] = "csv successfully uploaded"
+      redirect_to csv_edit_path
+    end
+
+    def export
+      solr = RSolr.connect url: Account.find_by(tenant: Apartment::Tenant.current).solr_endpoint.url
+      response = solr.get 'select', params: {
+          q: "member_of_collection_ids_ssim:#{params[:collection_id]}",
+          rows: 3400,
+          fl: "id"
+      }
+      unless response['response']['docs'].empty? || response['response']['docs'][0].empty?
+        work_ids = response['response']['docs'].map { |doc| doc['id'] }
+      end
+      #works    = ::ActiveFedora::Base.where member_of_collection_ids_ssim: params[:collection_id]
+      @csv_headers = ['type'] + work_fields
+      @csv_array   = [@csv_headers.join(',')]
+      work_ids.each do |work_id|
+        doc = ::SolrDocument.find work_id
+        add_line doc
+        doc._source[:file_set_ids_ssim].each do |file_id|
+          file_doc = ::SolrDocument.find file_id
+          add_line file_doc
+        end
+      end
+
+      send_data @csv_array.join("\n"),
+                :type => 'text/csv; charset=iso-8859-5; header=present',
+                :disposition => "attachment; filename=export.csv"
+    end
+
+    private
 
       def authenticate
         authorize! :create, available_works.first
       end
 
+      def add_line doc
+        line_hash = {}
+        line_hash['type'] = doc._source[:has_model_ssim].first
+        work_fields.each do |field|
+          line_hash[field] = create_cell doc, field
+        end
+        @csv_array << line_hash.values_at(*@csv_headers).map { |cell| cell = '' if cell.nil?; "\"#{cell.gsub("\"", "\"\"")}\"" }.join(',')
+
+      end
+
+      def work_fields
+        @fields ||=  available_works.map { |work| work.new.attributes.keys }.flatten.uniq - excluded_fields
+      end
+
+      def excluded_fields
+        %w[date_uploaded date_modified head tail state proxy_depositor on_behalf_of arkivo_checksum label
+       relative_path import_url part_of resource_type access_control_id
+       representative_id thumbnail_id rendering_ids admin_set_id embargo_id
+       lease_id]
+      end
+
+      def create_cell w, field
+        if field.include? 'date'
+          if w._source[field+'_tesim'].is_a?(Array)
+            w._source[field+'_tesim'].join('|')
+          else
+            w._source[field+'_tesim']
+          end
+        elsif w.respond_to?(field.to_sym)
+          if w.send(field).is_a?(Array)
+            w.send(field).join('|')
+          else
+            w.send(field)
+          end
+        end
+      end
+
       def available_works
         @available_works ||= Hyrax::QuickClassificationQuery.new(current_user).authorized_models
       end
 
       def parse_csv csv, mvs
-        csv = CSV.parse(File.read(csv), headers: true, encoding: 'utf-8').map(&:to_hash)
+        csv    = CSV.parse(File.read(csv), headers: true, encoding: 'utf-8').map(&:to_hash)
         @works = []
         csv.each do |row|
           type = row.first.last
@@ -115,19 +204,140 @@ module CdmMigrator
         end
       end
 
-      def check_paths csv_file
-        row_number = 1 # +1 offset to account for csv headers
-        @path_list = {}
+    def load_config
+      tenant = Account.find_by(tenant: Apartment::Tenant.current).cname
+      if CdmMigrator::Engine.config['tenant_settings'].has_key?(tenant)
+        settings = CdmMigrator::Engine.config['tenant_settings'][tenant]['csv_checker']
+        if settings.present?
+          # .map will throw an error if settings[key] has no value
+          @edtf_fields = settings['edtf_fields'].map(&:to_sym) if settings['edtf_fields']
+          @uri_fields = settings['valid_uri_fields'].map(&:to_sym) if settings['valid_uri_fields']
+          @separator = settings['multi_value_separator']
+          @separator_fields = settings['separator_fields'].map(&:to_sym) if settings['separator_fields']
+          @path_to_drive = settings['path_to_drive']
+        else
+          raise "Cdm Migrator couldn't find any configured settings. Are they in cdm_migrator.yml?"
+        end
+      else
+        raise "Cdm Migrator couldn't find this tenant. Is it configured?"
+      end
+    end
+
+    def check_csv csv_file
+      row_number = 1
+      @error_list = {}
+      check_mounted_drive if @path_to_drive.present?
+
+      CSV.foreach(csv_file, headers: true, header_converters: :symbol) do |row|
+        row_number +=1 # Tells user what CSV row the error is on
+        if row[:object_type].include? "Work"
+          check_edtf(row_number, row) if @edtf_fields.present?
+          check_uris(row_number, row) if @uri_fields.present?
+          if params[:multi_value_separator].present? and @separator_fields.present?
+            check_separator(row_number, row, params[:multi_value_separator])
+          else
+            alert_message = "No multi-value separator character was selected or no fields were configured. CSV Checker didn't check for valid separators."
+            if flash[:alert] and flash[:alert].exclude?(alert_message) # Only add this message once, rather than per line
+              flash[:alert] << alert_message
+            elsif flash[:alert].blank?
+              flash[:alert] = Array.wrap(alert_message)
+            end
+          end
+        elsif row[:object_type] == "File"
+          check_file_path(row_number, row[:url])
+        else
+          @error_list[row_number] = { "object_type" => "No or unknown object type. Please give a valid type (e.g. GenericWork, File)." }
+        end
+        @error_list.delete_if { |key, value| value.blank? } # Data are valid, no need to print the row
+      end
+    end
+
+    def check_mounted_drive
+      drive_address = @path_to_drive
+      unless Dir.exist?(drive_address) and !Dir[drive_address].empty?
+        flash[:alert] = "CSV Checker can't find the mounted drive to check file paths, so some paths may be mislabelled as incorrect. Please contact the administrator or try again later."
+      end
+    end
+
+    def check_file_path(row_number, file_path)
+      if file_path.nil?
+        @error_list[row_number] = { "url" => "url is blank." }
+      elsif File.file?(file_path.gsub("file://", "")) == false
+        @error_list[row_number] = { "url" => "No file found at #{file_path}" }
+      end
+    end
+
+    def check_edtf(row_number, row)
+      edtf_fields = @edtf_fields
+      edtf_errors = edtf_fields.each_with_object({}) do |field, hash|
+        temp_date = row[field]
+        # modify date so that the interval encompasses the years on the last interval date
+        temp_date = temp_date.gsub('/..','').gsub('%','?~').gsub(/\/$/,'')
+        date = temp_date.include?("/") ? temp_date.gsub(/([0-9]+X+\/)([0-9]+)(X+)/){"#{$1}"+"#{$2.to_i+1}"+"#{$3}"}.gsub("X","u") : temp_date
+        date = date.gsub("XX-","uu-").gsub("X-", "u-").gsub('XX?','uu').gsub('X?', 'u').gsub('u?','u').gsub('?','')
+        # edtf has trouble with year-month (e.g. "19uu-12") or year-season strings (e.g. "190u-23")
+        # that contain unspecified years, or intervals containing the above ("19uu-22/19uu-23", etc.).
+        # So we check for/create exceptions.
+        # Check for season interval
+        if Date.edtf(date) == nil and date != "unknown" # Accept season intervals
+          unless is_season?(date.split("/").first) and is_season?(date.split("/").second)
+            # If an interval then, check each date individually
+            if date.include?("/")
+              dates = date.split("/")
+            else
+              dates = [date]
+            end
+            #byebug
+            dates.each do |d|
+              # Dates with 'u' in the last digit of the year return invalid when in format YYYY-MM
+              # So we flub day specifity before checking again if the date is valid
+              unless Date.edtf(d + '-01') # Date.edtf('193u-03-01') returns valid
+                if match = d[/\d{3}u/] or match = d[/\d{2}u{2}-[2][1-4]/] # edtf can't parse single u in year (e.g. 192u) or uu in YYYY-SS (e.g. 19uu-21), so we replace it
+                  d.gsub!(match, match.gsub("u","0"))
+                  unless Date.edtf(d)
+                    hash[field.to_s] = "Blank or not a valid EDTF date."
+                  end
+                else
+                  hash[field.to_s] = "Blank or not a valid EDTF date."
+                end
+              end
+            end
+          end
+        end
+
+      end
+      @error_list[row_number] = edtf_errors
+    end
+
+    def is_season?(date)
+      Date.edtf(date).class == EDTF::Season
+    end
+
+    # <Example: should be http://rightsstatements.org/vocab/etc. NOT https://rightsstatements.org/page/etc.
+    def check_uris(row_number, row)
+      uri_fields = @uri_fields
+      uri_errors = uri_fields.each_with_object({}) do |field, hash|
+        if row[field].include? "page"
+          hash[field.to_s] = "Links to page instead of URI. (e.g. https://rightsstatements.org/page/etc. instead of http://rightsstatements.org/vocab/etc.)"
+        end
+      end
+      @error_list[row_number].merge!(uri_errors)
+    end
 
-        CSV.foreach(csv_file, headers: true, header_converters: :symbol) do |row|
-          row_number +=1 # Tells user what CSV row the bogus file path is on
-          next if row[:url].nil?
-          file_path = row[:url]
-          unless File.file?(file_path.gsub("file://", ""))
-            @path_list[row_number] = file_path
+    # Check multi-value separators
+    def check_separator(row_number, row, character)
+      uri_fields = @separator_fields
+      separator_errors = uri_fields.each_with_object({}) do |field, hash|
+        value = row[field]
+        if value.present?
+          URI.extract(value).each { |uri| value.gsub!(uri, '') }
+          unless value.split("").all? { |sep| sep == character } # Check if remaining characters are the correct separator
+            hash[field.to_s] = "May contain the wrong multi-value separator (i.e. not #{character})."
           end
         end
       end
+      @error_list[row_number].merge!(separator_errors)
+    end
 
       def default_page_title
         'CSV Batch Uploader'
@@ -139,8 +349,8 @@ module CdmMigrator
 
       def available_translations
         {
-          'en' => 'English',
-          'fr' => 'French'
+            'en' => 'English',
+            'fr' => 'French'
         }
       end
 
@@ -162,12 +372,12 @@ module CdmMigrator
       end
 
       def create_data data, type, object, mvs
-        final_data = {}
+        final_data     = {}
         accepted_terms = type.required_fields + secondary_terms(type)
         data.each do |key, att|
-          if(att.nil? || att.empty? || key.to_s.include?("object_type") || !accepted_terms.include?(key.to_sym) )
+          if (att.nil? || att.empty? || key.to_s.include?("object_type") || !accepted_terms.include?(key.to_sym))
             next
-          elsif(object.send(key).nil?)
+          elsif (object.send(key).nil?)
             final_data[key] = att
           else
             final_data[key] = att.split(mvs)
@@ -176,23 +386,23 @@ module CdmMigrator
         final_data
       end
 
-			def create_lease visibility, status_after, date
-				lease = Hydra::AccessControls::Lease.new(visibility_during_lease: visibility,
-							visibility_after_lease: status_after, lease_expiration_date: @lease_date)
-				lease.save
-			end
+      def create_lease visibility, status_after, date
+        lease = Hydra::AccessControls::Lease.new(visibility_during_lease: visibility,
+                                                 visibility_after_lease:  status_after, lease_expiration_date: @lease_date)
+        lease.save
+      end
 
-			def create_embargo visibility
-				embargo = Hydra::AccessControls::Embargo.new
-				embargo.visibility_during_embargo = visibility
-				embargo.visibility_after_embargo = @status_after
-				embargo.embargo_release_date = @embargo_date
-				embargo.save
-			end
+      def create_embargo visibility
+        embargo                           = Hydra::AccessControls::Embargo.new
+        embargo.visibility_during_embargo = visibility
+        embargo.visibility_after_embargo  = @status_after
+        embargo.embargo_release_date      = @embargo_date
+        embargo.save
+      end
 
       def log(user)
-          Hyrax::Operation.create!(user: user,
-                                              operation_type: "Attach Remote File")
+        Hyrax::Operation.create!(user:           user,
+                                 operation_type: "Attach Remote File")
       end
-	end
+  end
 end

data/app/jobs/cdm_migrator/batch_create_files_job.rb

@@ -2,7 +2,7 @@ module CdmMigrator
   class BatchCreateFilesJob < ActiveJob::Base
     queue_as Hyrax.config.ingest_queue_name
 
-    def perform work, ingest_work, user, last_work=false
+    def perform work, ingest_work, user
       ingest_work.files.each do |file|
         url = file[:url]
         last_file = ingest_work.files.last==file
@@ -11,10 +11,9 @@ module CdmMigrator
           actor = Hyrax::Actors::FileSetActor.new(fs, user)
           actor.create_metadata#(work, visibility: work.visibility)
           actor.attach_file_to_work(work)
-          #byebug
           fs.attributes = file[:metadata]
           fs.save!
-          CdmIngestFilesJob.perform_later(fs, url, user, ingest_work, last_file, last_work)
+          CdmIngestFilesJob.perform_later(fs, url, user, ingest_work, last_file)
         end
       end
     end

data/app/jobs/cdm_migrator/batch_create_works_job.rb

@@ -4,10 +4,9 @@ module CdmMigrator
 
     def perform(ingest, user)
       ingest.data.each do |w|
-        last_work = ingest.data.last==w
         ingest_work = IngestWork.new(w, ingest.id)
         ingest_work.save!
-        CreateWorkJob.perform_later ingest_work, user, ingest.admin_set_id, ingest.collection_id, last_work
+        CreateWorkJob.perform_later ingest_work, user, ingest.admin_set_id, ingest.collection_id
       end
 
     end

data/app/jobs/cdm_migrator/cdm_ingest_files_job.rb

@@ -2,7 +2,7 @@ module CdmMigrator
   class CdmIngestFilesJob < ActiveJob::Base
     queue_as Hyrax.config.ingest_queue_name
 
-    def perform(fs, url, user, ingest_work = nil, last_file = false, last_work = false)
+    def perform(fs, url, user, ingest_work = nil, last_file = false)
       if url.include?("http") && File.extname(url).include?("pdf")
         download = open(url)
         dir = Rails.root.join('public', 'uploads', 'csv_pdfs')
@@ -16,14 +16,14 @@ module CdmMigrator
         IO.copy_stream(download, url)
         url = "file://"+url.to_s
       end
-      uri = URI.parse(url.gsub(' ','%20'))
+      uri = URI.parse(url.gsub(' ','%20').gsub(/[\[\]@#\$\*{}]/, ""))
       if uri.scheme == 'file'
-        IngestLocalFileJob.perform_now(fs, uri.path.gsub('%20',' '), user)
+        IngestLocalFileJob.perform_now(fs, url.gsub('file://',''), user)
       else
+        URI.parse(url.gsub(' ','%20'))
         ImportUrlJob.perform_now(fs, log(user))
       end
       ingest_work.update_attribute('complete', true) if last_file
-      BatchIngest.find(ingest_work.id).update_attribute('complete', true) if last_work
     end
 
     def log(user)

data/app/jobs/cdm_migrator/create_work_job.rb

@@ -2,18 +2,24 @@ module CdmMigrator
   class CreateWorkJob < ActiveJob::Base
     queue_as Hyrax.config.ingest_queue_name
 
-    def perform(ingest_work, user, admin_set_id, collection_id, last_work=false)
+    def perform(ingest_work, user, admin_set_id, collection_id)
       admin_set = ::AdminSet.find(admin_set_id) rescue nil
       collection = ::Collection.find(collection_id) rescue nil
       work = Object.const_get(ingest_work.work_type).new
       #status_after, embargo_date, lease_date = nil, nil, nil
       work.apply_depositor_metadata(user)
       work.attributes = ingest_work.data
+      if ingest_work.data.has_key? 'downloadable'
+        # Convert string to boolean
+        work.downloadable = ActiveModel::Type::Boolean.new.cast(ingest_work.data['downloadable'])
+      elsif work.attributes.include? 'downloadable' # Set work to downloadable by default
+        work.downloadable = true
+      end
       work.member_of_collections = [collection] if collection
       work.admin_set = admin_set if admin_set
       work.date_uploaded = DateTime.now
       work.save
-      BatchCreateFilesJob.perform_later work, ingest_work, user, last_work
+      BatchCreateFilesJob.perform_later work, ingest_work, user
 
     end
   end

data/app/jobs/cdm_migrator/update_object_job.rb

@@ -0,0 +1,10 @@
+module CdmMigrator
+  class UpdateObjectJob < ActiveJob::Base
+
+
+    def perform(attributes)
+      obj = ActiveFedora::Base.find
+
+    end
+  end
+end

data/app/models/cdm_migrator/batch_ingest.rb

@@ -7,11 +7,14 @@ module CdmMigrator
     end
 
     def progress
-      if complete?
+      return "Complete" if complete?
+      completed = IngestWork.where(batch_ingest_id: id, complete: true ).length
+      if completed==data.length
+        complete=true
+        save
         "Complete"
       else
-        completed = IngestWork.where(batch_ingest_id: id, complete: true ).length.to_s
-        "#{completed}/#{size}"
+        "#{completed.to_s}/#{size}"
       end
     end
 
@@ -20,7 +23,7 @@ module CdmMigrator
     end
 
     def complete?
-      self.complete
+      complete
     end
 
     def message?

data/app/views/cdm_migrator/cdm/mappings.html.erb

@@ -30,7 +30,7 @@ border:1px solid black;
     </tr>
   <% end %>
   </table>
-  <%= select_tag "mappings_url", options_for_select(@dirs) if @cdm_dirs %>
+  <%= select_tag "mappings_url", options_for_select(@dirs.sort { |x,y| x[0].downcase <=> y[0].downcase }) if @cdm_dirs %>
   <%= hidden_field_tag "work", params[:work] %>
   <%= submit_tag 'generate csv'%>
 <% end %>

data/app/views/cdm_migrator/csv/_error_list.html.erb

@@ -0,0 +1,21 @@
+<% if @error_list && @error_list.any? %>
+  <%#= @error_list.inspect %>
+  <table class="table table-striped">
+    <thead>
+    <tr>
+      <th scope="col" style="min-width: 100px;">Line No.</th>
+      <th scope="col" style="margin-right: 0.5em;">Column(s)</th>
+      <th scope="col">Issue</th>
+    </tr>
+    </thead>
+    <tbody>
+    <% @error_list.keys.each do |line_number| %>
+      <tr>
+        <td><%= line_number %></td>
+        <td><%= @error_list[line_number].keys.join("<br />").html_safe %></td>
+        <td><%= @error_list[line_number].values.join("<br />").html_safe %></td>
+      </tr>
+    <% end %>
+    </tbody>
+  </table>
+<% end %>

data/app/views/cdm_migrator/csv/csv_checker.html.erb

@@ -0,0 +1,46 @@
+<% provide :page_title, "CSV Checker" %>
+
+<%#= flash[:alert].join("<br/>") if flash[:alert] %>
+
+<h1>CSV Checker</h1>
+
+<!-- <div class="row"> -->
+<p>This tool validates CSV data and creates a table listing any errors.</p>
+
+<a role="button" class="collapse-toggle collapsed csv-collapse-link" data-toggle="collapse" data-target="#errors-explanation" aria-expanded="false">
+  <span id="csv-collapse-link-text">Expand for more details</span>
+</a>
+<div class="collapse" id="errors-explanation">
+  <ul>
+    <li><strong>File paths: </strong>The url field contains a valid path to a file.</li>
+    <li><strong>Multi-value separator: </strong>Configured fields with URIs contain the right separator character (e.g. |).</li>
+    <% if @path_to_drive.present? %>
+      <li><strong>Mounted drive: </strong>A mounted directory (folder) exists and is not empty.</li>
+    <% end %>
+    <% if @edtf_fields.present? %>
+      <li><strong>EDTF dates: </strong>Configured fields contain valid <a href="https://www.loc.gov/standards/datetime/" target="_blank">EDTF</a> dates or "unknown."</li>
+    <% end %>
+    <% if @uri_fields.present? %>
+      <li><strong>Valid URIs ("page" vs "vocab"): </strong>Configured fields with URIs link to the "vocab" address rather than the "page" address. For example, rights_statement should be "http://rightsstatement.org/vocab/..." and not "https://rightsstatement.org/page/..."</li>
+    <% end %>
+  </ul>
+</div>
+
+
+
+<%= form_tag(check_csv_path, remote: true, method: :post, multipart: true, id: "csv-form") do %>
+  <div class="input-group">
+    <%= label_tag :multi_value_separator %>
+    <%= text_field_tag(:multi_value_separator, @separator, size: 1) %>
+    <%= file_field_tag(:file, class: "form-control-file") %>
+    <%= hidden_field_tag :authenticity_token, value: form_authenticity_token %>
+    <%= button_tag(type: :submit, class: "btn btn-large btn-primary", style: "margin-top: 1em;") do %>
+      Check CSV
+    <% end %>
+<% end %>
+</div>
+<!-- </div> -->
+
+<div id="error_list" class="col-md-10 offset-md-1">
+  <%= render 'error_list' %>
+</div>

data/app/views/cdm_migrator/csv/edit.html.erb

@@ -0,0 +1,18 @@
+
+<h1><span class="fa fa-map"></span> Export </h1>
+
+<%= form_tag main_app.csv_export_path, method: :post do %>
+  <%= select_tag 'collection_id', options_for_select(@collections), include_blank: true %>
+  <%= submit_tag "Download CSV", class: 'btn btn-primary' %>
+<% end %>
+
+<h1><span class="fa fa-map"></span> Update </h1>
+
+<%= form_for :csv_update, url: csv_update_path do |f| %>
+  <%= f.label 'Multi-value Separator:' %>
+  <%= f.text_field 'mvs' %>
+  <br />
+  <%= f.file_field 'csv_file' %>
+  <br />
+  <%= f.submit 'Save' %>
+<% end %>

data/config/routes.rb

@@ -5,9 +5,12 @@ Rails.application.routes.draw do
   get '/cdm_migrator/my/batches', to: 'cdm_migrator/csv#index', as: 'csv_my_batches'
   get '/cdm_migrator/batches', to: 'cdm_migrator/csv#index', as: 'csv_all_batches'
   get '/cdm_migrator/rerun/:id', to: 'cdm_migrator/csv#rerun', as: 'csv_rerun'
+  get '/cdm_migrator/edit', to: 'cdm_migrator/csv#edit', as: 'csv_edit'
+  post '/cdm_migrator/export', to: 'cdm_migrator/csv#export', as: 'csv_export'
+  post '/cdm_migrator/update', to: 'cdm_migrator/csv#update', as: 'csv_update'
 
-  get '/cdm_migrator/file_path_checker', to: 'cdm_migrator/csv#file_path_checker', as: 'file_path_checker'
-  post '/cdm_migrator/file_path_checker', to: 'cdm_migrator/csv#file_path_checker', as: 'check_file_paths'
+  get '/cdm_migrator/csv_checker', to: 'cdm_migrator/csv#csv_checker', as: 'csv_checker'
+  post '/cdm_migrator/csv_checker', to: 'cdm_migrator/csv#csv_checker', as: 'check_csv'
 
   get '/cdm_migrator/collection', to: 'cdm_migrator/cdm#collection', as: 'cdm_start'
   post '/cdm_migrator/mappings/', to: 'cdm_migrator/cdm#mappings', as: 'cdm_mappings'

data/lib/cdm_migrator/version.rb

@@ -1,3 +1,3 @@
 module CdmMigrator
-  VERSION = '3.0.1'
+  VERSION = '3.2.1'
 end

data/lib/generators/cdm_migrator/install/install_generator.rb

@@ -22,8 +22,8 @@ class CdmMigrator::InstallGenerator < Rails::Generators::Base
             "			<%= menu.nav_link(main_app.cdm_start_path) do %>\n" \
             "				<span class=\"fa fa-map\"></span> <span class=\"sidebar-action-text\"><%= t('CDM Mapping Tool') %></span>\n" \
             "			<% end %>\n" \
-            "     <%= menu.nav_link(main_app.file_path_checker_path) do %>\n" \
-            "       <span class=\"fa fa-check-circle\"></span><span>File Path Checker</span>\n" \
+            "     <%= menu.nav_link(main_app.csv_checker_path) do %>\n" \
+            "       <span class=\"fa fa-check-circle\"></span><span>CSV Checker</span>\n" \
             "     <% end %>\n" \
             "			<%= menu.nav_link(main_app.csv_upload_path) do %>\n"\
             "				<span class=\"fa fa-angle-double-up\"></span> <span class=\"sidebar-action-text\"><%= t('CSV Batch Uploader') %></span>\n" \
@@ -67,5 +67,13 @@ class CdmMigrator::InstallGenerator < Rails::Generators::Base
   def inject_content_dm_yml
     copy_file("config/cdm_migrator.yml", "config/cdm_migrator.yml") unless File.file?("config/cdm_migrator.yml")
   end
+
+  def inject_stylesheets
+    css_file_path = "app/assets/stylesheets/application.css"
+    copy_file("stylesheets/csv_checker.css", "app/assets/stylesheets/csv_checker.css") unless File.file?("app/assets/styelsheets/csv_checker.css")
+    insert_into_file css_file_path, :before => " *= require_self\n" do
+      " *= require csv_checker\n "
+    end
+  end
   
 end

data/lib/generators/cdm_migrator/install/templates/config/cdm_migrator.yml

@@ -1,25 +1,38 @@
-cdm_api:
+tenant_settings:
   tenant1.institution.com:
-    url: 'http://your-content-dm-host'
-    port: 8080
-    type: 'front'
-    dirs:
-      dir1: '/dir1/path/goes/here'
-      dir2: '/dir2/path/goes/here'
+    cdm_api:
+      url: 'http://your-content-dm-host'
+      port: 8080
+      type: 'front'
+      dirs:
+        dir1: '/dir1/path/goes/here'
+        dir2: '/dir2/path/goes/here'
   tenant2.institution.com:
-    url: 'http://your-content-dm-host'
-    port: 8080
-    type: 'front'
-    dirs:
-      dir1: '/dir1/path/goes/here'
-      dir2: '/dir2/path/goes/here'
+    cdm_api:
+      url: 'http://your-content-dm-host'
+      port: 8080
+      type: 'front'
+      dirs:
+        dir1: '/dir1/path/goes/here'
+        dir2: '/dir2/path/goes/here'
   default:
-    url: 'http://your-content-dm-host'
-    port: 8080
-    type: 'front'
-    dirs:
-      dir1: '/dir1/path/goes/here'
-      dir2: '/dir2/path/goes/here'
+    cdm_api:
+      url: 'http://your-content-dm-host'
+      port: 8080
+      type: 'front'
+      dirs:
+        dir1: '/dir1/path/goes/here'
+        dir2: '/dir2/path/goes/here'
+    csv_checker:
+      edtf_fields:
+      # - date_created
+      valid_uri_fields:
+      # - rights_statement
+      # - genre
+      separator_fields:
+      # - subject
+      multi_value_separator: # '|'
+      path_to_drive: #'/mnt/drive'
 default_fields:
   - title
   - creator

data/lib/generators/cdm_migrator/install/templates/sidebar/_tasks.html.erb

@@ -8,8 +8,8 @@
 				<span class="fa fa-map"></span> <span class="sidebar-action-text"><%= t('CDM Mapping Tool') %></span>
 			<% end %>
 
-      <%= menu.nav_link(main_app.file_path_checker_path) do %>
-        <span class="fa fa-check-circle"></span><span class="sidebar-action-text"><%= t('File Path Checker') %></span>
+      <%= menu.nav_link(main_app.csv_checker_path) do %>
+        <span class="fa fa-check-circle"></span><span class="sidebar-action-text"><%= t('CSV Checker') %></span>
       <% end %>
 
 			<%= menu.nav_link(main_app.csv_upload_path) do %>
@@ -19,6 +19,10 @@
 			<%= menu.nav_link(main_app.csv_my_batches_path) do %>
 				<span class="fa fa-database"></span> <span class="sidebar-action-text"><%= t('Batches') %></span>
 			<% end %>
+
+			<%= menu.nav_link(main_app.csv_edit_path) do %>
+				<span class="fa fa-angle-double-up"></span> <span class="sidebar-action-text"><%= t('Batch Update') %></span>
+			<% end %>
 		<% end %>
 	</li>
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cdm_migrator
 version: !ruby/object:Gem::Version
-  version: 3.0.1
+  version: 3.2.1
 platform: ruby
 authors:
 - sephirothkod
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-12-20 00:00:00.000000000 Z
+date: 2020-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -81,6 +81,7 @@ files:
 - app/assets/config/cdm_migrator_manifest.js
 - app/assets/javascripts/cdm_migrator/application.js
 - app/assets/stylesheets/cdm_migrator/application.css
+- app/assets/stylesheets/cdm_migrator/csv_checker.css
 - app/controllers/cdm_migrator/application_controller.rb
 - app/controllers/cdm_migrator/cdm_controller.rb
 - app/controllers/cdm_migrator/csv_controller.rb
@@ -90,6 +91,7 @@ files:
 - app/jobs/cdm_migrator/batch_create_works_job.rb
 - app/jobs/cdm_migrator/cdm_ingest_files_job.rb
 - app/jobs/cdm_migrator/create_work_job.rb
+- app/jobs/cdm_migrator/update_object_job.rb
 - app/mailers/cdm_migrator/application_mailer.rb
 - app/models/cdm_migrator/application_record.rb
 - app/models/cdm_migrator/batch_ingest.rb
@@ -98,11 +100,11 @@ files:
 - app/views/cdm_migrator/cdm/mappings.html.erb
 - app/views/cdm_migrator/csv/_batches_list.html.erb
 - app/views/cdm_migrator/csv/_default_group.html.erb
+- app/views/cdm_migrator/csv/_error_list.html.erb
 - app/views/cdm_migrator/csv/_list_batches.html.erb
-- app/views/cdm_migrator/csv/_path_list.html.erb
-- app/views/cdm_migrator/csv/_results_pagination.html.erb
 - app/views/cdm_migrator/csv/_tabs.html.erb
-- app/views/cdm_migrator/csv/file_path_checker.html.erb
+- app/views/cdm_migrator/csv/csv_checker.html.erb
+- app/views/cdm_migrator/csv/edit.html.erb
 - app/views/cdm_migrator/csv/index.html.erb
 - app/views/cdm_migrator/csv/upload.html.erb
 - app/views/layouts/cdm_migrator/application.html.erb
@@ -122,7 +124,7 @@ homepage: https://github.com/UVicLibrary/cdm_migrator
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -137,9 +139,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: ContentDM to Hyrax migrator.
 test_files: []

data/app/views/cdm_migrator/csv/_path_list.html.erb

@@ -1,19 +0,0 @@
-<% if @path_list && @path_list.any? %>
-    <table class="table table-striped">
-      <thead>
-        <tr>
-          <th scope="col" style="min-width: 100px;">Line No.</th>
-          <th scope="col">File Path (url)</th>
-        </tr>
-      </thead>
-      <tbody>
-        <% @path_list.each do |line, path| %>
-          <tr>
-            <td><%= line %></td>
-            <td><%= path %></td>
-          </tr>
-        <% end %>
-      </tbody>
-    </table>
-<% end %>
-

data/app/views/cdm_migrator/csv/_results_pagination.html.erb

@@ -1,9 +0,0 @@
-<% if @response.total_pages > 1 %>
- <div class="row record-padding">
-  <div class="col-md-9">
-    <div class="pagination">
-      <%= paginate @response, outer_window: 2, theme: 'blacklight', route_set: hyrax %>
-    </div>
-  </div>
- </div>
-<% end %>

data/app/views/cdm_migrator/csv/file_path_checker.html.erb

@@ -1,21 +0,0 @@
-<% provide :page_title, "File Path Checker" %>
-
-<h1>File Path Checker</h1>
-
-<!-- <div class="row"> -->
-<p>This tool checks if a file exists at each url in a csv.</p>
-<%= form_tag(check_file_paths_path, remote: true, method: :post, multipart: true, id: "csv-form") do %>
-<div class="input-group">
-  <%= file_field_tag(:file, class: "form-control-file") %>
-  <%= hidden_field_tag :authenticity_token, value: form_authenticity_token %>
-  <%= button_tag(type: :submit, class: "btn btn-large btn-primary", style: "margin-top: 1em;") do %>
-    Check CSV
-  <% end %>
-<% end %>
-</div>
-<!-- </div> -->
-
-<div id="path_list" class="col-md-10 offset-md-1">
-  <%= render 'path_list' %>
-</div>
-