cdm_migrator 3.5.2 → 4.6.2

data/app/jobs/cdm_migrator/create_work_job.rb

@@ -42,7 +42,7 @@ module CdmMigrator
 
       permissions = permissions_config.map do |permission_level, group_name|
         # Check if the permission level is configured in Hyrax
-        raise "#{permission_level} permission is not configured in this repo. Is it set in Hyrax.config.permission_levels?" unless Hyrax.config.permission_levels.index(permission_level).presence
+        raise "#{permission_level} permission is not configured in this repo. Is it set in Hyrax.config.permission_levels?" unless Hyrax.config.permission_levels.has_value?(permission_level)
         # Construct each permission as a hash
         group_name.map do |group|
           { name: group, type: "group", access: permission_level }

data/app/models/cdm_migrator/batch_ingest.rb

@@ -1,33 +1,33 @@
-module CdmMigrator
-  class BatchIngest < ActiveRecord::Base
-    serialize :data
-
-    def name
-      csv.split('/').last.gsub(/[0-9]{10}/,"")
-    end
-
-    def progress
-      return "Complete" if complete?
-      completed = IngestWork.where(batch_ingest_id: id, complete: true ).length
-      if completed==data.length
-        complete=true
-        save
-        "Complete"
-      else
-        "#{completed.to_s}/#{size}"
-      end
-    end
-
-    def username
-      @username ||= User.find(user_id).name
-    end
-
-    def complete?
-      complete
-    end
-
-    def message?
-      not(message.nil?||message.empty?)
-    end
-  end
-end
+module CdmMigrator
+  class BatchIngest < ActiveRecord::Base
+    serialize :data, coder: YAML
+
+    def name
+      csv.split('/').last.gsub(/[0-9]{10}/,"")
+    end
+
+    def progress
+      return "Complete" if complete?
+      completed = IngestWork.where(batch_ingest_id: id, complete: true ).length
+      if completed==data.length
+        complete=true
+        save
+        "Complete"
+      else
+        "#{completed.to_s}/#{size}"
+      end
+    end
+
+    def username
+      @username ||= User.find(user_id).name
+    end
+
+    def complete?
+      complete
+    end
+
+    def message?
+      not(message.nil?||message.empty?)
+    end
+  end
+end

data/app/models/cdm_migrator/csv_checker.rb

@@ -0,0 +1,195 @@
+module CdmMigrator
+  class CsvChecker
+    
+    def initialize
+      load_config
+    end
+    
+    # @param[String] - the path to the CSV
+    # @return [Hash] - where hash['errors'] is a list of errors found in the csv,
+    #                  and hash['alerts'] are warnings to display to the user
+    def check_csv(csv, multi_value_separator = "|")
+      @csv = csv
+      row_number = 1
+
+      results = {}
+      @error_list = {}
+      @alerts = []
+
+      check_mounted_drive if @path_to_drive.present?
+      
+      CSV.foreach(csv, headers: true, header_converters: :symbol) do |row|
+        row_number +=1 # Tells user what CSV row the error is on
+        if row[:object_type].try(:include?, "Work")
+          check_dates(row_number, row) if @date_fields.present?
+          check_uris(row_number, row) if @uri_fields.present?
+          if @separator_fields.present?
+            check_multi_val_fields(row_number, row, multi_value_separator)
+          else
+            alert_message = "No multi-value separator character was selected or no fields were configured. CSV Checker didn't check for valid separators."
+            if @alerts.exclude?(alert_message) # Only add this message once, rather than per line
+             @alerts << alert_message
+            end
+          end
+        elsif row[:object_type] == "File"
+          check_file_path(row_number, row[:url])
+          check_transcript_length(row_number, row[:transcript]) if row[:transcript].present?
+          check_file_size(row_number, row[:url])
+        else
+          @error_list[row_number] = { "object_type" => "No or unknown object type. Please give a valid type (e.g. GenericWork, File)." }
+          @error_list = @error_list.delete_if { |_, value| value.blank? } # Data are valid, no need to print the row
+        end
+      end
+      results['errors'] = @error_list
+      results['alerts'] = @alerts
+      results
+    end
+    
+    private
+
+    def work_form(worktype = "GenericWork")
+      Module.const_get("Hyrax::#{worktype}Form") rescue nil || Module.const_get("Hyrax::Forms::WorkForm")
+    end
+
+    def file_form
+      Module.const_get("Hyrax::FileSetForm") rescue nil || Module.const_get("Hyrax::Forms::FileSetEditForm")
+    end
+    
+    def check_file_path(row_number, file_path)
+      if file_path.nil?
+        @error_list[row_number] = { "url" => "url is blank." }
+      elsif File.file?(file_path.gsub("file://", "")) == false
+        @error_list[row_number] = { "url" => "No file found at #{file_path}" }
+      end
+    end
+    
+    def check_mounted_drive
+      drive_address = @path_to_drive
+      unless Dir.exist?(drive_address) and !Dir[drive_address].empty?
+        @alerts << "CSV Checker can't find the mounted drive to check file paths, so some paths may be mislabelled as incorrect. Please contact the administrator or try again later."
+      end
+    end
+
+    def check_dates(row_number, row)
+      date_fields = @date_fields
+      unless @date_indexing_service
+        @alerts << "No date indexing service was configured so CSV Checker didn't validate dates."
+        return
+      end
+      edtf_errors = date_fields.each_with_object({}) do |field, hash|
+        next unless row[field]
+        begin
+          @date_indexing_service.new(row[field])
+        rescue *@date_indexing_service.error_classes => error
+          hash[field.to_s] = "#{error.message}"
+        end
+      end
+      @error_list[row_number] = edtf_errors
+    end
+
+    # <Example: should be http://rightsstatements.org/vocab/etc. NOT https://rightsstatements.org/page/etc.
+    def check_uris(row_number, row)
+      uri_fields = @uri_fields
+      uri_errors = uri_fields.each_with_object({}) do |field, hash|
+        if row[field] and row[field].include? "page"
+          hash[field.to_s] = "Links to page instead of URI. (e.g. https://rightsstatements.org/page/etc. instead of http://rightsstatements.org/vocab/etc.)"
+        elsif row[field] and row[field].match?("https://vocab.getty")
+          hash[field.to_s] = "Getty AAT URIs should use http instead of https"
+        elsif field == :language
+          unless row[field].match?("http://id.loc.gov/vocabulary/iso639-3")
+            hash[field.to_s] = "Value doesn't look like a Library of Congress ISO639-3 URI. Is it a URI that starts with http://id.loc.gov/vocabulary/iso639-3?"
+          end
+        end
+      end
+      if @error_list[row_number].present?
+        @error_list[row_number].merge!(uri_errors)
+      else
+        @error_list[row_number] = uri_errors
+      end
+    end
+
+    # Check multi-value separators
+    def check_multi_val_fields(row_number, row, character)
+      uri_fields = @separator_fields
+      separator_errors = uri_fields.each_with_object({}) do |field, hash|
+        if value = row[field]
+          # Check for leading or trailing spaces
+          if value.match %r{ #{Regexp.escape(character)}|#{Regexp.escape(character)} }
+            hash[field.to_s] = "Contains leading or trailing whitespace around multi-value separator."
+          end
+          values = value.split(character).map(&:strip)
+          values.each do |val|
+            if val.match(URI::RFC2396_PARSER.make_regexp) # Val should be URI
+              remainder = val.gsub(val.match(URI::RFC2396_PARSER.make_regexp)[0],'')
+              unless remainder.blank?
+                hash[field.to_s] = "May contain the wrong multi-value separator or a typo in the URI."
+              end
+              if field != :genre && field != :resource_type
+                unless val.match(/\bhttps?:\/\/id.worldcat.org\/fast\/\d+\b/)
+                  hash[field.to_s] = "Field may contain an invalid URI or is missing a separator between URIs."
+                end
+              end
+            else # Or val should be string
+              invalid_chars = ["\\"]
+              # Make exceptions for backslashes that are part of whitespace characters
+              # by deleting them before checking for stray \s
+              if val.delete("\t\r\n\s\n").match Regexp.union(invalid_chars)
+                hash[field.to_s] = "May contain an invalid character such as #{invalid_chars.to_sentence(last_word_connector: ", or ")}."
+              end
+            end
+          end
+        end
+      end
+      @error_list[row_number].merge!(separator_errors)
+    end
+    
+    def check_file_size(row_number, file_path)
+      if file_path.present? && File.file?(file_path) && @max_file_size
+        if File.size(file_path.gsub("file://", "")) > @max_file_size
+          @error_list[row_number] = { "file size" => "The file at #{file_path} is too large to be uploaded. Please compress the file or split it into parts.
+                                                  Each part should be under #{helpers.number_to_human_size(@max_file_size)}." }
+        end
+      end
+    end
+
+    def check_transcript_length(row_number, transcript)
+      if transcript.is_a? String
+        if transcript.length > 9000
+          @error_list[row_number] = { "transcript" => "Transcript is too long (over 9000 characters)." }
+        end
+      elsif transcript.is_a? Array
+        if transcript.any? { |tr| tr.length > 9000 }
+          @error_list[row_number] = { "transcript" => "Transcript is too long (over 9000 characters)." }
+        end
+      end
+    end
+    
+    def load_config
+      if ENV['HYKU_MULTITENANT']
+        tenant = Account.find_by(tenant: Apartment::Tenant.current).cname
+      else
+        tenant = "default"
+      end
+      
+      if CdmMigrator::Engine.config['tenant_settings'].has_key?(tenant)
+        settings = CdmMigrator::Engine.config['tenant_settings'][tenant]['csv_checker']
+        if settings.present?
+          @date_indexing_service = settings['date_indexing_service'].first.constantize if settings['date_indexing_service']
+          @date_fields = settings['date_fields'].map(&:to_sym) if settings['date_fields']
+          @uri_fields = settings['valid_uri_fields'].map(&:to_sym) if settings['valid_uri_fields']
+          @separator = settings['multi_value_separator']
+          @separator_fields = settings['separator_fields'].map(&:to_sym) if settings['separator_fields']
+          @path_to_drive = settings['path_to_drive']
+          # If you would like to change this to match the uploader's max file size,
+          # change this to Hyrax.config.uploader[:maxFileSize]
+          @max_file_size = settings['max_file_size']
+        else
+          raise "Cdm Migrator couldn't find any configured settings. Are they in cdm_migrator.yml?"
+        end
+      else
+        raise "Cdm Migrator couldn't find this tenant. Is it configured?"
+      end
+    end
+    
+  end
+end

data/app/models/cdm_migrator/ingest_work.rb

@@ -1,7 +1,7 @@
-module CdmMigrator
-  class IngestWork < ApplicationRecord
-
-    serialize :data
-    serialize :files
-  end
-end
+module CdmMigrator
+  class IngestWork < ApplicationRecord
+
+    serialize :data, coder: YAML
+    serialize :files, coder: YAML
+  end
+end

data/app/views/cdm_migrator/csv/csv_checker.html.erb

@@ -1,46 +1,39 @@
-<% provide :page_title, "CSV Checker" %>
-
-<%#= flash[:alert].join("<br/>") if flash[:alert] %>
-
-<h1>CSV Checker</h1>
-
-<!-- <div class="row"> -->
-<p>This tool validates CSV data and creates a table listing any errors.</p>
-
-<a role="button" class="collapse-toggle collapsed csv-collapse-link" data-toggle="collapse" data-target="#errors-explanation" aria-expanded="false">
-  <span id="csv-collapse-link-text">Expand for more details</span>
-</a>
-<div class="collapse" id="errors-explanation">
-  <ul>
-    <li><strong>File paths: </strong>The url field contains a valid path to a file.</li>
-    <li><strong>Multi-value separator: </strong>Configured fields with URIs contain the right separator character (e.g. |).</li>
-    <% if @path_to_drive.present? %>
-      <li><strong>Mounted drive: </strong>A mounted directory (folder) exists and is not empty.</li>
-    <% end %>
-    <% if @edtf_fields.present? %>
-      <li><strong>EDTF dates: </strong>Configured fields contain valid <a href="https://www.loc.gov/standards/datetime/" target="_blank">EDTF</a> dates or "unknown."</li>
-    <% end %>
-    <% if @uri_fields.present? %>
-      <li><strong>Valid URIs ("page" vs "vocab"): </strong>Configured fields with URIs link to the "vocab" address rather than the "page" address. For example, rights_statement should be "http://rightsstatement.org/vocab/..." and not "https://rightsstatement.org/page/..."</li>
-    <% end %>
-  </ul>
-</div>
-
-
-
-<%= form_tag(check_csv_path, remote: true, method: :post, multipart: true, id: "csv-form") do %>
-  <div class="input-group">
-    <%= label_tag :multi_value_separator %>
-    <%= text_field_tag(:multi_value_separator, @separator, size: 1) %>
-    <%= file_field_tag(:file, class: "form-control-file") %>
-    <%= hidden_field_tag :authenticity_token, value: form_authenticity_token %>
-    <%= button_tag(type: :submit, class: "btn btn-large btn-primary", style: "margin-top: 1em;") do %>
-      Check CSV
-    <% end %>
-<% end %>
-</div>
-<!-- </div> -->
-
-<div id="error_list" class="col-md-10 offset-md-1">
-  <%= render 'error_list' %>
-</div>
+<% content_for(:title, "CSV Checker") %>
+<h1 class="mt-4 mb-3">CSV Checker</h1>
+
+<p>This tool validates CSV data and creates a table listing any errors.</p>
+
+<a role="button" class="collapse-toggle collapsed csv-collapse-link" data-toggle="collapse" data-target="#errors-explanation" aria-expanded="false">
+  <span id="csv-collapse-link-text">Expand for more details</span>
+</a>
+<div class="collapse" id="errors-explanation">
+  <ul>
+    <li><strong>File paths: </strong>The url field contains a valid path to a file.</li>
+    <li><strong>Multi-value separator: </strong>Configured fields with URIs contain the right separator character (e.g. |).</li>
+    <% if @path_to_drive.present? %>
+      <li><strong>Mounted drive: </strong>A mounted directory (folder) exists and is not empty.</li>
+    <% end %>
+    <% if @edtf_fields.present? %>
+      <li><strong>EDTF dates: </strong>Configured fields contain valid <a href="https://www.loc.gov/standards/datetime/" target="_blank">EDTF</a> dates or "unknown."</li>
+    <% end %>
+    <% if @uri_fields.present? %>
+      <li><strong>Valid URIs ("page" vs "vocab"): </strong>Configured fields with URIs link to the "vocab" address rather than the "page" address. For example, rights_statement should be "http://rightsstatement.org/vocab/..." and not "https://rightsstatement.org/page/..."</li>
+    <% end %>
+  </ul>
+</div>
+
+<%= form_tag(check_csv_path, method: :post, multipart: true, local: true, id: "csv-form") do %>
+  <div class="input-group">
+    <%= label_tag :multi_value_separator %>
+    <%= text_field_tag(:multi_value_separator, @separator, size: 1) %>
+    <%= file_field_tag(:file, class: "form-control-file") %>
+    <%= hidden_field_tag :authenticity_token, value: form_authenticity_token %>
+    <%= button_tag(type: :submit, class: "btn btn-large btn-primary", style: "margin-top: 1em;") do %>
+      Check CSV
+    <% end %>
+<% end %>
+</div>
+
+<div id="error_list" class="col-md-10 offset-md-1">
+  <%= render 'error_list' %>
+</div>

data/app/views/cdm_migrator/csv/edit.html.erb

@@ -1,17 +1,36 @@
-
-<h1><span class="fa fa-map"></span> Export </h1>
-
-<%= form_tag main_app.csv_export_path, method: :post do %>
-  <%= select_tag 'collection_id', options_for_select(@collections.sort { |x,y| x[0].downcase <=> y[0].downcase }), include_blank: true %>
-  <%= submit_tag "Download CSV", class: 'btn btn-primary' %>
-<% end %>
-<h1><span class="fa fa-map"></span> Update </h1>
-
-<%= form_for :csv_update, url: csv_update_path do |f| %>
-  <%= f.label 'Multi-value Separator:' %>
-  <%= f.text_field 'mvs' %>
-  <br />
-  <%= f.file_field 'csv_file' %>
-  <br />
-  <%= f.submit 'Save' %>
-<% end %>
+
+<h1 class="mt-3"><span class="fa fa-download"></span> Export Metadata</h1>
+
+<%= simple_form_for :csv_export, url: csv_export_path, method: :post do |f| %>
+  <div class="form-row form-inline mb-3">
+    <%= f.label :collection, label: 'Collection', class: 'col-form-label', for: 'export-collection', required: false  %>
+    <%= f.input_field :collection,
+                      id: "export-collection",
+                      class: 'col-md-4 form-control',
+                      placeholder: 'Select a collection...',
+                      inline_label: 'Collection',
+                      data: {
+                        autocomplete: 'collection',
+                        'autocomplete-url' => Rails.application.routes.url_helpers.qa_path + '/search/collections?access=deposit'
+                      } %>
+  </div>
+  <%= f.submit "Download CSV", class: 'btn btn-primary mb-2', data: { disable_with: false } %>
+<% end %>
+
+<h1 class="mt-3"><span class="fa fa-cloud-upload"></span> Batch Update Metadata</h1>
+
+<%= form_for :csv_update, url: csv_update_path do |f| %>
+  <%= f.label 'Multi-value Separator:' %>
+  <%= f.text_field 'mvs', class: "mb-2", value: '|'  %>
+  <br />
+  <%= f.file_field 'csv_file', class: "mb-2" %>
+  <br />
+  <%= f.submit 'Save', class: 'btn btn-primary' %>
+<% end %>
+
+<script>
+  // Enable autocomplete on the collection id form
+  autocomplete = new autocompleteModule()
+  element = $('#export-collection')
+  autocomplete.setup(element, element.data('autocomplete'), element.data('autocomplete-url'))
+</script>

data/app/views/cdm_migrator/csv/upload.html.erb

@@ -1,18 +1,52 @@
-<h1><span class="fa fa-angle-double-up"></span> <%= default_page_title %></h1>
-
-<%= form_for :csv_import, url: csv_create_path do |f| %>
-	<%= f.label 'Multi-value Separator:' %>
-    <%= f.text_field 'mvs' %>
-    <br />
-    <%= f.file_field 'csv_file' %>
-    <br />
-    <%= f.label 'Admin Set:' %>
-    <%= select_tag "admin_set", options_for_select(@admin_sets), :include_blank => true %>
-    <br />
-    <%= f.label 'Collection:' %>
-    <%= select_tag "collection", options_for_select(@collections.sort { |x,y| x[0].downcase <=> y[0].downcase }), :include_blank => true %>
-		<br />
-	<%= f.submit 'Save', data: { disable_with: 'Submitting CSV...' } %>
-<% end %>
-<br />
-<%= link_to "template", csv_generate_path %>
+<% content_for(:title, "CSV Batch Upload") %>
+<h1 class="mt-4 mb-3">CSV Batch Upload</h1>
+
+<div class="form-group col-md-6">
+  <%= simple_form_for :csv_import, url: csv_create_path do |f| %>
+
+    <div class="form-row form-inline mb-3">
+      <%= f.label :mvs, label: 'Multi-value Separator', class: 'col-form-label', required: false %>
+      <%= f.input_field :mvs, class: 'form-control col-md-1', value: '|' %>
+    </div>
+
+    <div class="form-row form-inline mb-3">
+      <%= f.label :csv_file, class: 'col-form-label sr-only', required: false %>
+      <%= f.file_field :csv_file, class: 'form-control-file col-md-4' %>
+    </div>
+
+    <div class="form-row form-inline mb-3">
+      <%= f.label :admin_set, class: 'col-form-label', required: false  %>
+      <%= f.input :admin_set,
+                  collection: @admin_sets,
+                  label: false,
+                  class: 'form-inline' %>
+    </div>
+
+    <div class="form-row form-inline mb-3">
+      <%= f.label :collection, label: 'Collection', class: 'col-form-label', for: 'batch-upload-select-collection', required: false  %>
+      <%= f.input_field :collection,
+                        id: "batch-upload-select-collection",
+                        class: 'col-md-8 form-control',
+                        placeholder: 'Select a collection...',
+                        inline_label: 'Collection',
+                        data: {
+                          autocomplete: 'collection',
+                          'autocomplete-url' => Rails.application.routes.url_helpers.qa_path + '/search/collections?access=deposit'
+                        } %>
+    </div>
+
+
+    <%= f.submit 'Save',
+                 class: 'btn btn-primary',
+                 data: { disable_with: 'Submitting CSV...' } %>
+    <%= link_to "template", csv_generate_path, class: 'ml-4' %>
+  <% end %>
+</div>
+
+
+
+<script>
+    autocomplete = new autocompleteModule()
+    element = $('#batch-upload-select-collection')
+    autocomplete.setup(element, element.data('autocomplete'), element.data('autocomplete-url'))
+</script>

data/db/migrate/20191212192315_create_ingest_works.rb

@@ -1,18 +1,20 @@
-class CreateIngestWorks < ActiveRecord::Migration[5.0]
-  def up
-    create_table :ingest_works do |t|
-      t.string :work_type
-      t.text :data
-      t.text :files
-      t.boolean :complete, :default => false
-
-      t.references :batch_ingest, foreign_key: true
-
-      t.timestamps
-    end
-  end
-
-  def down
-    drop_table :ingest_works
-  end
-end
+class CreateIngestWorks < ActiveRecord::Migration[5.0]
+  def up
+    unless table_exists?(:ingest_works)
+      create_table :ingest_works do |t|
+        t.string :work_type
+        t.text :data
+        t.text :files
+        t.boolean :complete, :default => false
+
+        t.references :batch_ingest, foreign_key: true
+
+        t.timestamps
+      end
+    end
+  end
+
+  def down
+    drop_table :ingest_works
+  end
+end

data/lib/cdm_migrator/version.rb

@@ -1,3 +1,3 @@
 module CdmMigrator
-  VERSION = '3.5.2'
+  VERSION = '4.6.2'
 end