cdm_migrator 3.1.0 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cdf5a7cb1cb95ce144dd931ed6b309ebc20d5f93ee8e91e6cfbabafded6a35fb
-  data.tar.gz: 6594d9da22ee42db51c96ac5393289b210c5b60e2e0a3066aa8f5e3571d6987e
+  metadata.gz: 330afa0ea8f2de397dc822fafef651778e039b0b422ec8ab93e61bd4ed6e9ced
+  data.tar.gz: 6860866888bf49430fbbb4eace6e38bb7b2da09f6b8633608e7508cac374dc4c
 SHA512:
-  metadata.gz: 2798dd9b01e019d298cf007604fbb54105856e1761eb7d5cca62b7bd1d1f4587b4b48b0d43e9881ffb0a01ebf8d4a27b7b3469d83a68c265353368ebca4b2489
-  data.tar.gz: 0ab6fa9e3a5324fb0c80692cdcda02aee0fb18dd0b0d20a570789e3478292987a91101cbd86e362d929c56d197af6ed20f82372d2b275ecd69f1f4a5dda3f076
+  metadata.gz: f263a7da82234a60ac6ac4f73529b226762b483dee9ce093712f8dd3c34dc97b3b1b96b5b62cbc98d0de5f2dc1157367b5c1352a5cb79bd7703f45a707ceb034
+  data.tar.gz: f4f095574c88c63970d2de9032ad96ad134ff34952b78068a2ddceee404d9998134ccfac15ee29ca75d2ac9c7b8f4a3a4db33030b6218d8b100afa64ea357cd3

data/README.md

@@ -27,11 +27,12 @@ to insert the yml and add a link to your Hyrax dashboard
 
 ## Usage
 1. Add your ContentDM url and api port to the cdm_migrator.yml file.
-2. Navigate to the *cdm_migrator/collection* url to select your contentdm collection and what type of work you want to export it to and click "choose mappings".
-3. Map the ContentDM fields to your Hyrax work and file fields\* and click "generate CSV".
-4. Refine the CSV as you see fit.
-5. Navigate to the *cdm_migrator/upload* url; choose your multi-value seperator (default is |) and upload your CSV file.
-6. Done.
+2. Configure the CSV Checker with the appropriate fields, paths, or multi-value separator (in cdm_migrator.yml).
+3. Navigate to the *cdm_migrator/collection* url to select your contentdm collection and what type of work you want to export it to and click "choose mappings".
+4. Map the ContentDM fields to your Hyrax work and file fields\* and click "generate CSV".
+5. Refine the CSV as you see fit. Optional: go to *cdm_migrator/csv_checker* and upload it to validate metadata fields and/or file paths.
+6. Navigate to the *cdm_migrator/upload* url; choose your multi-value separator (default is |) and upload your CSV file.
+7. Done.
 
 \* cdm_migrator uses the generated Hyrax forms (ex. Hyrax::Forms::GenericWorkForm) in your host application to obtain it's terms for mapping. If you have added terms to your FileSet model extend the Hyrax::Forms::FileSetEditForm with Hyrax::FileSetForm in your host application so that the changes will be detected by the migrator. You can also add a list of fields in the yml file, under "default fields".
 

data/app/assets/stylesheets/cdm_migrator/csv_checker.css

@@ -0,0 +1,37 @@
+.csv-collapse-link.collapsed {
+    display: inline-block;
+    vertical-align: top;
+}
+
+#csv-collapse-link-text::after {
+    content: "❯";
+    display: inline-block;
+    font-size: smaller;
+    right: 15px;
+    transform: rotate(90deg);
+    margin-left: 0.5em;
+}
+
+.collapsed #csv-collapse-link-text::after {
+    transform: rotate(0deg);
+    transition: transform 0.1s ease;
+    margin-left: 2px;
+    vertical-align: top;
+}
+
+#errors-explanation ul li {
+    list-style-type: none;
+    margin-bottom: 2px;
+}
+
+#errors-explanation ul {
+    padding-left: 0;
+}
+
+#errors-explanation {
+    margin-top: 5px;
+}
+
+#csv-form {
+    margin-top: 1em;
+}

data/app/controllers/cdm_migrator/cdm_controller.rb

@@ -104,10 +104,10 @@ module CdmMigrator
 
 		def load_yaml
 			stripped_url = request.base_url.dup.gsub(/https?:\/\//, '').gsub(/:[0-9]*/,'')
-			if CdmMigrator::Engine.config['cdm_api'].key? stripped_url
-				tenant = CdmMigrator::Engine.config['cdm_api'][stripped_url]
+			if CdmMigrator::Engine.config['tenant_settings'].key? stripped_url
+				tenant = CdmMigrator::Engine.config['tenant_settings'][stripped_url]['cdm_api']
 			else
-				tenant = CdmMigrator::Engine.config['cdm_api']['default']
+				tenant = CdmMigrator::Engine.config['tenant_settings']['default']['cdm_api']
 			end
 			@cdm_url = tenant['url']
 			@cdm_port = tenant['port']

data/app/controllers/cdm_migrator/csv_controller.rb

@@ -4,15 +4,15 @@ module CdmMigrator
     include ActionView::Helpers::UrlHelper
     layout 'hyrax/dashboard' if Hyrax
     before_action :authenticate, except: :index
+    before_action :load_config, only: :csv_checker
 
-    def file_path_checker
+    def csv_checker
       if params[:file]
-        check_paths params[:file].path
-
-        if @path_list.blank?
-          flash[:notice] = "All file paths are valid."
+        check_csv params[:file].path
+        if @error_list.blank?
+          flash[:notice] = "All data are valid."
         else
-          flash[:error] = "Cdm Migrator couldn't find files at the following urls. Please correct the paths and try again."
+          flash[:error] = "The CSV Checker found some errors in the CSV. Please correct them and check again."
         end
       end
     end
@@ -41,9 +41,9 @@ module CdmMigrator
       File.open(csv, 'wb') do |file|
         file.write(params[:csv_import][:csv_file].read)
       end
-      check_paths csv
-      if @path_list.present?
-        flash[:error] = "some file paths are invalid"
+      check_csv csv
+      if @error_list.present?
+        flash[:error] = "Cdm Migrator found some problems with the CSV. Use the CSV Checker for more details."
       end
       parse_csv(csv, params[:csv_import][:mvs])
 
@@ -203,19 +203,104 @@ module CdmMigrator
         end
       end
 
-      def check_paths csv_file
-        row_number = 1 # +1 offset to account for csv headers
-        @path_list = {}
+    def load_config
+      tenant = Account.find_by(tenant: Apartment::Tenant.current).cname
+      if CdmMigrator::Engine.config['tenant_settings'].has_key?(tenant)
+        settings = CdmMigrator::Engine.config['tenant_settings'][tenant]['csv_checker']
+        if settings.present?
+          # .map will throw an error if settings[key] has no value
+          @edtf_fields = settings['edtf_fields'].map(&:to_sym) if settings['edtf_fields']
+          @uri_fields = settings['valid_uri_fields'].map(&:to_sym) if settings['valid_uri_fields']
+          @separator = settings['multi_value_separator']
+          @separator_fields = settings['separator_fields'].map(&:to_sym) if settings['separator_fields']
+          @path_to_drive = settings['path_to_drive']
+        else
+          raise "Cdm Migrator couldn't find any configured settings. Are they in cdm_migrator.yml?"
+        end
+      else
+        raise "Cdm Migrator couldn't find this tenant. Is it configured?"
+      end
+    end
+
+    def check_csv csv_file
+      row_number = 1
+      @error_list = {}
+      check_mounted_drive if @path_to_drive.present?
+
+      CSV.foreach(csv_file, headers: true, header_converters: :symbol) do |row|
+        row_number +=1 # Tells user what CSV row the error is on
+        if row[:object_type].include? "Work"
+          check_edtf(row_number, row) if @edtf_fields.present?
+          check_uris(row_number, row) if @uri_fields.present?
+          if params[:multi_value_separator].present? and @separator_fields.present?
+            check_separator(row_number, row, params[:multi_value_separator])
+          else
+            alert_message = "No multi-value separator character was selected or no fields were configured. CSV Checker didn't check for valid separators."
+            if flash[:alert] and flash[:alert].exclude?(alert_message) # Only add this message once, rather than per line
+              flash[:alert] << alert_message
+            elsif flash[:alert].blank?
+              flash[:alert] = Array.wrap(alert_message)
+            end
+          end
+        elsif row[:object_type] == "File"
+          check_file_path(row_number, row[:url])
+        else
+          @error_list[row_number] = { "object_type" => "No or unknown object type. Please give a valid type (e.g. GenericWork, File)." }
+        end
+        @error_list.delete_if { |key, value| value.blank? } # Data are valid, no need to print the row
+      end
+    end
+
+    def check_mounted_drive
+      drive_address = @path_to_drive
+      unless Dir.exist?(drive_address) and !Dir[drive_address].empty?
+        flash[:alert] = "CSV Checker can't find the mounted drive to check file paths, so some paths may be mislabelled as incorrect. Please contact the administrator or try again later."
+      end
+    end
+
+    def check_file_path(row_number, file_path)
+      if file_path.nil?
+        @error_list[row_number] = { "url" => "url is blank." }
+      elsif File.file?(file_path.gsub("file://", "")) == false
+        @error_list[row_number] = { "url" => "No file found at #{file_path}" }
+      end
+    end
+
+    def check_edtf(row_number, row)
+      edtf_fields = @edtf_fields
+      edtf_errors = edtf_fields.each_with_object({}) do |field, hash|
+        if Date.edtf(row[field]) == nil and row[field] != "unknown"
+          hash[field.to_s] = "Blank or not a valid EDTF date."
+        end
+      end
+      @error_list[row_number] = edtf_errors
+    end
+
+    # <Example: should be http://rightsstatements.org/vocab/etc. NOT https://rightsstatements.org/page/etc.
+    def check_uris(row_number, row)
+      uri_fields = @uri_fields
+      uri_errors = uri_fields.each_with_object({}) do |field, hash|
+        if row[field].include? "page"
+          hash[field.to_s] = "Links to page instead of URI. (e.g. https://rightsstatements.org/page/etc. instead of http://rightsstatements.org/vocab/etc.)"
+        end
+      end
+      @error_list[row_number].merge!(uri_errors)
+    end
 
-        CSV.foreach(csv_file, headers: true, header_converters: :symbol) do |row|
-          row_number += 1 # Tells user what CSV row the bogus file path is on
-          next if row[:url].nil?
-          file_path = row[:url]
-          unless File.file?(file_path.gsub("file://", ""))
-            @path_list[row_number] = file_path
+    # Check multi-value separators
+    def check_separator(row_number, row, character)
+      uri_fields = @separator_fields
+      separator_errors = uri_fields.each_with_object({}) do |field, hash|
+        value = row[field]
+        if value.present?
+          URI.extract(value).each { |uri| value.gsub!(uri, '') }
+          unless value.split("").all?(character) # Check if remaining characters are the correct separator
+            hash[field.to_s] = "May contain the wrong multi-value separator (i.e. not #{character})."
           end
         end
       end
+      @error_list[row_number].merge!(separator_errors)
+    end
 
       def default_page_title
         'CSV Batch Uploader'

data/app/jobs/cdm_migrator/create_work_job.rb

@@ -9,6 +9,12 @@ module CdmMigrator
       #status_after, embargo_date, lease_date = nil, nil, nil
       work.apply_depositor_metadata(user)
       work.attributes = ingest_work.data
+      if ingest_work.data.has_key? 'downloadable'
+        # Convert string to boolean
+        work.downloadable = ActiveModel::Type::Boolean.new.cast(ingest_work.data['downloadable'])
+      elsif work.attributes.include? 'downloadable' # Set work to downloadable by default
+        work.downloadable = true
+      end
       work.member_of_collections = [collection] if collection
       work.admin_set = admin_set if admin_set
       work.date_uploaded = DateTime.now

data/app/views/cdm_migrator/csv/csv_checker.html.erb

@@ -0,0 +1,46 @@
+<% provide :page_title, "CSV Checker" %>
+
+<%#= flash[:alert].join("<br/>") if flash[:alert] %>
+
+<h1>CSV Checker</h1>
+
+<!-- <div class="row"> -->
+<p>This tool validates CSV data and creates a table listing any errors.</p>
+
+<a role="button" class="collapse-toggle collapsed csv-collapse-link" data-toggle="collapse" data-target="#errors-explanation" aria-expanded="false">
+  <span id="csv-collapse-link-text">Expand for more details</span>
+</a>
+<div class="collapse" id="errors-explanation">
+  <ul>
+    <li><strong>File paths: </strong>The url field contains a valid path to a file.</li>
+    <li><strong>Multi-value separator: </strong>Configured fields with URIs contain the right separator character (e.g. |).</li>
+    <% if @path_to_drive.present? %>
+      <li><strong>Mounted drive: </strong>A mounted directory (folder) exists and is not empty.</li>
+    <% end %>
+    <% if @edtf_fields.present? %>
+      <li><strong>EDTF dates: </strong>Configured fields contain valid <a href="https://www.loc.gov/standards/datetime/" target="_blank">EDTF</a> dates or "unknown."</li>
+    <% end %>
+    <% if @uri_fields.present? %>
+      <li><strong>Valid URIs ("page" vs "vocab"): </strong>Configured fields with URIs link to the "vocab" address rather than the "page" address. For example, rights_statement should be "http://rightsstatement.org/vocab/..." and not "https://rightsstatement.org/page/..."</li>
+    <% end %>
+  </ul>
+</div>
+
+
+
+<%= form_tag(check_csv_path, remote: true, method: :post, multipart: true, id: "csv-form") do %>
+  <div class="input-group">
+    <%= label_tag :multi_value_separator %>
+    <%= text_field_tag(:multi_value_separator, @separator, size: 1) %>
+    <%= file_field_tag(:file, class: "form-control-file") %>
+    <%= hidden_field_tag :authenticity_token, value: form_authenticity_token %>
+    <%= button_tag(type: :submit, class: "btn btn-large btn-primary", style: "margin-top: 1em;") do %>
+      Check CSV
+    <% end %>
+<% end %>
+</div>
+<!-- </div> -->
+
+<div id="error_list" class="col-md-10 offset-md-1">
+  <%= render 'error_list' %>
+</div>

data/config/routes.rb

@@ -9,8 +9,8 @@ Rails.application.routes.draw do
   post '/cdm_migrator/export', to: 'cdm_migrator/csv#export', as: 'csv_export'
   post '/cdm_migrator/update', to: 'cdm_migrator/csv#update', as: 'csv_update'
 
-  get '/cdm_migrator/file_path_checker', to: 'cdm_migrator/csv#file_path_checker', as: 'file_path_checker'
-  post '/cdm_migrator/file_path_checker', to: 'cdm_migrator/csv#file_path_checker', as: 'check_file_paths'
+  get '/cdm_migrator/csv_checker', to: 'cdm_migrator/csv#csv_checker', as: 'csv_checker'
+  post '/cdm_migrator/csv_checker', to: 'cdm_migrator/csv#csv_checker', as: 'check_csv'
 
   get '/cdm_migrator/collection', to: 'cdm_migrator/cdm#collection', as: 'cdm_start'
   post '/cdm_migrator/mappings/', to: 'cdm_migrator/cdm#mappings', as: 'cdm_mappings'

data/lib/cdm_migrator/version.rb

@@ -1,3 +1,3 @@
 module CdmMigrator
-  VERSION = '3.1.0'
+  VERSION = '3.2.0'
 end

data/lib/generators/cdm_migrator/install/install_generator.rb

@@ -22,8 +22,8 @@ class CdmMigrator::InstallGenerator < Rails::Generators::Base
             "			<%= menu.nav_link(main_app.cdm_start_path) do %>\n" \
             "				<span class=\"fa fa-map\"></span> <span class=\"sidebar-action-text\"><%= t('CDM Mapping Tool') %></span>\n" \
             "			<% end %>\n" \
-            "     <%= menu.nav_link(main_app.file_path_checker_path) do %>\n" \
-            "       <span class=\"fa fa-check-circle\"></span><span>File Path Checker</span>\n" \
+            "     <%= menu.nav_link(main_app.csv_checker_path) do %>\n" \
+            "       <span class=\"fa fa-check-circle\"></span><span>CSV Checker</span>\n" \
             "     <% end %>\n" \
             "			<%= menu.nav_link(main_app.csv_upload_path) do %>\n"\
             "				<span class=\"fa fa-angle-double-up\"></span> <span class=\"sidebar-action-text\"><%= t('CSV Batch Uploader') %></span>\n" \
@@ -67,5 +67,13 @@ class CdmMigrator::InstallGenerator < Rails::Generators::Base
   def inject_content_dm_yml
     copy_file("config/cdm_migrator.yml", "config/cdm_migrator.yml") unless File.file?("config/cdm_migrator.yml")
   end
+
+  def inject_stylesheets
+    css_file_path = "app/assets/stylesheets/application.css"
+    copy_file("stylesheets/csv_checker.css", "app/assets/stylesheets/csv_checker.css") unless File.file?("app/assets/styelsheets/csv_checker.css")
+    insert_into_file css_file_path, :before => " *= require_self\n" do
+      " *= require csv_checker\n "
+    end
+  end
   
 end

data/lib/generators/cdm_migrator/install/templates/config/cdm_migrator.yml

@@ -1,25 +1,38 @@
-cdm_api:
+tenant_settings:
   tenant1.institution.com:
-    url: 'http://your-content-dm-host'
-    port: 8080
-    type: 'front'
-    dirs:
-      dir1: '/dir1/path/goes/here'
-      dir2: '/dir2/path/goes/here'
+    cdm_api:
+      url: 'http://your-content-dm-host'
+      port: 8080
+      type: 'front'
+      dirs:
+        dir1: '/dir1/path/goes/here'
+        dir2: '/dir2/path/goes/here'
   tenant2.institution.com:
-    url: 'http://your-content-dm-host'
-    port: 8080
-    type: 'front'
-    dirs:
-      dir1: '/dir1/path/goes/here'
-      dir2: '/dir2/path/goes/here'
+    cdm_api:
+      url: 'http://your-content-dm-host'
+      port: 8080
+      type: 'front'
+      dirs:
+        dir1: '/dir1/path/goes/here'
+        dir2: '/dir2/path/goes/here'
   default:
-    url: 'http://your-content-dm-host'
-    port: 8080
-    type: 'front'
-    dirs:
-      dir1: '/dir1/path/goes/here'
-      dir2: '/dir2/path/goes/here'
+    cdm_api:
+      url: 'http://your-content-dm-host'
+      port: 8080
+      type: 'front'
+      dirs:
+        dir1: '/dir1/path/goes/here'
+        dir2: '/dir2/path/goes/here'
+    csv_checker:
+      edtf_fields:
+      # - date_created
+      valid_uri_fields:
+      # - rights_statement
+      # - genre
+      separator_fields:
+      # - subject
+      multi_value_separator: # '|'
+      path_to_drive: #'/mnt/drive'
 default_fields:
   - title
   - creator

data/lib/generators/cdm_migrator/install/templates/sidebar/_tasks.html.erb

@@ -8,8 +8,8 @@
 				<span class="fa fa-map"></span> <span class="sidebar-action-text"><%= t('CDM Mapping Tool') %></span>
 			<% end %>
 
-      <%= menu.nav_link(main_app.file_path_checker_path) do %>
-        <span class="fa fa-check-circle"></span><span class="sidebar-action-text"><%= t('File Path Checker') %></span>
+      <%= menu.nav_link(main_app.csv_checker_path) do %>
+        <span class="fa fa-check-circle"></span><span class="sidebar-action-text"><%= t('CSV Checker') %></span>
       <% end %>
 
 			<%= menu.nav_link(main_app.csv_upload_path) do %>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cdm_migrator
 version: !ruby/object:Gem::Version
-  version: 3.1.0
+  version: 3.2.0
 platform: ruby
 authors:
 - sephirothkod
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-19 00:00:00.000000000 Z
+date: 2020-08-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -81,6 +81,7 @@ files:
 - app/assets/config/cdm_migrator_manifest.js
 - app/assets/javascripts/cdm_migrator/application.js
 - app/assets/stylesheets/cdm_migrator/application.css
+- app/assets/stylesheets/cdm_migrator/csv_checker.css
 - app/controllers/cdm_migrator/application_controller.rb
 - app/controllers/cdm_migrator/cdm_controller.rb
 - app/controllers/cdm_migrator/csv_controller.rb
@@ -103,8 +104,8 @@ files:
 - app/views/cdm_migrator/csv/_path_list.html.erb
 - app/views/cdm_migrator/csv/_results_pagination.html.erb
 - app/views/cdm_migrator/csv/_tabs.html.erb
+- app/views/cdm_migrator/csv/csv_checker.html.erb
 - app/views/cdm_migrator/csv/edit.html.erb
-- app/views/cdm_migrator/csv/file_path_checker.html.erb
 - app/views/cdm_migrator/csv/index.html.erb
 - app/views/cdm_migrator/csv/upload.html.erb
 - app/views/layouts/cdm_migrator/application.html.erb

data/app/views/cdm_migrator/csv/file_path_checker.html.erb

@@ -1,21 +0,0 @@
-<% provide :page_title, "File Path Checker" %>
-
-<h1>File Path Checker</h1>
-
-<!-- <div class="row"> -->
-<p>This tool checks if a file exists at each url in a csv.</p>
-<%= form_tag(check_file_paths_path, remote: true, method: :post, multipart: true, id: "csv-form") do %>
-<div class="input-group">
-  <%= file_field_tag(:file, class: "form-control-file") %>
-  <%= hidden_field_tag :authenticity_token, value: form_authenticity_token %>
-  <%= button_tag(type: :submit, class: "btn btn-large btn-primary", style: "margin-top: 1em;") do %>
-    Check CSV
-  <% end %>
-<% end %>
-</div>
-<!-- </div> -->
-
-<div id="path_list" class="col-md-10 offset-md-1">
-  <%= render 'path_list' %>
-</div>
-