cdm_migrator 3.2.1 → 3.3.2

data/app/helpers/cdm_migrator/application_helper.rb

@@ -1,4 +1,4 @@
-module CdmMigrator
-  module ApplicationHelper
-  end
-end
+module CdmMigrator
+  module ApplicationHelper
+  end
+end

data/app/jobs/cdm_migrator/application_job.rb

@@ -1,4 +1,4 @@
-module CdmMigrator
-  class ApplicationJob < ActiveJob::Base
-  end
-end
+module CdmMigrator
+  class ApplicationJob < ActiveJob::Base
+  end
+end

data/app/jobs/cdm_migrator/batch_create_files_job.rb

@@ -1,21 +1,33 @@
-module CdmMigrator
-  class BatchCreateFilesJob < ActiveJob::Base
-    queue_as Hyrax.config.ingest_queue_name
-
-    def perform work, ingest_work, user
-      ingest_work.files.each do |file|
-        url = file[:url]
-        last_file = ingest_work.files.last==file
-        ::FileSet.new(import_url: url, label: file[:title]) do |fs|
-          fs.save
-          actor = Hyrax::Actors::FileSetActor.new(fs, user)
-          actor.create_metadata#(work, visibility: work.visibility)
-          actor.attach_file_to_work(work)
-          fs.attributes = file[:metadata]
-          fs.save!
-          CdmIngestFilesJob.perform_later(fs, url, user, ingest_work, last_file)
-        end
-      end
-    end
-  end
+module CdmMigrator
+  class BatchCreateFilesJob < ActiveJob::Base
+    queue_as Hyrax.config.ingest_queue_name
+
+    def perform work, ingest_work, user
+      unless work.ordered_members.to_a.empty?
+        # This only executes if the job failed before adding any file sets and is now retrying
+        ordered_members = work.ordered_members.to_a
+        # If the last file set's file was never ingested, do that now
+        last_file_set = ordered_members.last
+        if last_file_set.present? and last_file_set.files.none?
+          CdmMigrator::CdmIngestFilesJob.perform_later(last_file_set, last_file_set.import_url, user)
+        end
+        files = ingest_work.files[ordered_members.length..-1]
+      else
+        files = ingest_work.files
+      end
+      files.each do |file|
+        url = file[:url]
+        last_file = ingest_work.files.last==file
+        ::FileSet.new(import_url: url, label: file[:title]) do |fs|
+          fs.save
+          actor = Hyrax::Actors::FileSetActor.new(fs, user)
+          actor.create_metadata
+          actor.attach_file_to_work(work)
+          fs.attributes = file[:metadata]
+          fs.save!
+          CdmIngestFilesJob.perform_later(fs, url, user, ingest_work, last_file)
+        end
+      end
+    end
+  end
 end

data/app/jobs/cdm_migrator/batch_create_files_with_ordered_members_job.rb

@@ -0,0 +1,45 @@
+module CdmMigrator
+  class BatchCreateFilesWithOrderedMembersJob < ActiveJob::Base
+    queue_as Hyrax.config.ingest_queue_name
+
+    # From documentation at https://tinyurl.com/nh4c5e9j:
+    #     When adding member {FileSetBehavior}s to a {WorkBehavior}, {Hyrax} saves
+    #     and reloads the work for each new member FileSet. This can significantly
+    #     slow down ingest for Works with many member FileSets. The saving and
+    #     reloading happens in {FileSetActor#attach_to_work}.
+    #     See the url for more details.
+
+    #     With this job, the member association is saved once
+    #     at the very end instead, which can speed up the upload process. This job takes
+    #     advantage of the same strategy as Hyrax::Actors::FileSetOrderedMembersActor
+    #     but you don't need the OrderedMembersActor constant initialized.
+
+    # This rescue is a safeguard against creating lots of orphan file sets if there
+    # are recurring errors (see https://tinyurl.com/nh4c5e9j). Instead, CdmMigrator
+    # will fall back to creating file sets one-by-one if this job fails once.
+    rescue_from(StandardError) do |exception|
+      Rails.logger.error "BatchCreateFilesWithOrderedMembersJob error: #{exception.to_s}"
+      RestartUploadFromMiddleJob.perform_later(arguments[0], arguments[1], arguments[2])
+    end
+
+    def perform work, ingest_work, user
+        ordered_members = []
+        ingest_work.files.each do |file|
+          url = file[:url]
+          last_file = ingest_work.files.last==file
+          ::FileSet.new(import_url: url, label: file[:title]) do |fs|
+            fs.attributes = file[:metadata]
+            fs.save!
+            ordered_members << fs
+          end
+        end
+        actor = Hyrax::Actors::OrderedMembersActor.new(ordered_members, user)
+        actor.attach_ordered_members_to_work(work)
+        work.representative = work.ordered_members.to_a.first
+        work.thumbnail_id = work.ordered_member_ids.first
+        work.save!
+        work.file_sets.each { |fs| CdmIngestFilesJob.perform_later(fs, fs.import_url, user, ingest_work) }
+    end
+    
+  end
+end

data/app/jobs/cdm_migrator/batch_create_works_job.rb

@@ -1,14 +1,20 @@
-module CdmMigrator
-  class BatchCreateWorksJob < ActiveJob::Base
-    queue_as Hyrax.config.ingest_queue_name
-
-    def perform(ingest, user)
-      ingest.data.each do |w|
-        ingest_work = IngestWork.new(w, ingest.id)
-        ingest_work.save!
-        CreateWorkJob.perform_later ingest_work, user, ingest.admin_set_id, ingest.collection_id
-      end
-
-    end
-  end
-end
+module CdmMigrator
+  class BatchCreateWorksJob < ActiveJob::Base
+    queue_as Hyrax.config.ingest_queue_name
+
+    def perform(ingest, user)
+      ingest.data.each do |w|
+        ingest_work = IngestWork.new({
+           work_type: w[:type],
+           data: w[:metadata],
+           files: w[:files],
+           batch_ingest_id: ingest.id
+        })
+        ingest_work.save!
+        # puts ingest_work.inspect
+        CreateWorkJob.perform_later ingest_work, user, ingest.admin_set_id, ingest.collection_id
+      end
+
+    end
+  end
+end

data/app/jobs/cdm_migrator/cdm_ingest_files_job.rb

@@ -1,35 +1,35 @@
-module CdmMigrator
-  class CdmIngestFilesJob < ActiveJob::Base
-    queue_as Hyrax.config.ingest_queue_name
-
-    def perform(fs, url, user, ingest_work = nil, last_file = false)
-      if url.include?("http") && File.extname(url).include?("pdf")
-        download = open(url)
-        dir = Rails.root.join('public', 'uploads', 'csv_pdfs')
-        FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
-        filename = download.base_uri.to_s.split('/').last
-        url = dir.join(filename)
-        if fs.title.empty?
-          fs.title << filename.split('.').first
-          fs.save
-        end
-        IO.copy_stream(download, url)
-        url = "file://"+url.to_s
-      end
-      uri = URI.parse(url.gsub(' ','%20').gsub(/[\[\]@#\$\*{}]/, ""))
-      if uri.scheme == 'file'
-        IngestLocalFileJob.perform_now(fs, url.gsub('file://',''), user)
-      else
-        URI.parse(url.gsub(' ','%20'))
-        ImportUrlJob.perform_now(fs, log(user))
-      end
-      ingest_work.update_attribute('complete', true) if last_file
-    end
-
-    def log(user)
-      Hyrax::Operation.create!(user: user,
-                               operation_type: "Attach Remote File")
-    end
-
-  end
-end
+module CdmMigrator
+  class CdmIngestFilesJob < ActiveJob::Base
+    queue_as Hyrax.config.ingest_queue_name
+
+    def perform(fs, url, user, ingest_work = nil, last_file = false)
+      if url.include?("http") && File.extname(url).include?("pdf")
+        download = open(url)
+        dir = Rails.root.join('public', 'uploads', 'csv_pdfs')
+        FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
+        filename = download.base_uri.to_s.split('/').last
+        url = dir.join(filename)
+        if fs.title.empty?
+          fs.title << filename.split('.').first
+          fs.save
+        end
+        IO.copy_stream(download, url)
+        url = "file://"+url.to_s
+      end
+      uri = URI.parse(url.gsub(' ','%20').gsub(/[\[\]@#\$\*{}]/, ""))
+      if uri.scheme == 'file'
+        IngestLocalFileJob.perform_now(fs, url.gsub('file://',''), user)
+      else
+        URI.parse(url.gsub(' ','%20'))
+        ImportUrlJob.perform_now(fs, log(user))
+      end
+      ingest_work.update_attribute('complete', true) if last_file
+    end
+
+    def log(user)
+      Hyrax::Operation.create!(user: user,
+                               operation_type: "Attach Remote File")
+    end
+
+  end
+end

data/app/jobs/cdm_migrator/create_work_job.rb

@@ -1,26 +1,37 @@
-module CdmMigrator
-  class CreateWorkJob < ActiveJob::Base
-    queue_as Hyrax.config.ingest_queue_name
-
-    def perform(ingest_work, user, admin_set_id, collection_id)
-      admin_set = ::AdminSet.find(admin_set_id) rescue nil
-      collection = ::Collection.find(collection_id) rescue nil
-      work = Object.const_get(ingest_work.work_type).new
-      #status_after, embargo_date, lease_date = nil, nil, nil
-      work.apply_depositor_metadata(user)
-      work.attributes = ingest_work.data
-      if ingest_work.data.has_key? 'downloadable'
-        # Convert string to boolean
-        work.downloadable = ActiveModel::Type::Boolean.new.cast(ingest_work.data['downloadable'])
-      elsif work.attributes.include? 'downloadable' # Set work to downloadable by default
-        work.downloadable = true
-      end
-      work.member_of_collections = [collection] if collection
-      work.admin_set = admin_set if admin_set
-      work.date_uploaded = DateTime.now
-      work.save
-      BatchCreateFilesJob.perform_later work, ingest_work, user
-
-    end
-  end
+module CdmMigrator
+  class CreateWorkJob < ActiveJob::Base
+    queue_as Hyrax.config.ingest_queue_name
+
+    def perform(ingest_work, user, admin_set_id, collection_id)
+      admin_set = ::AdminSet.find(admin_set_id) rescue nil
+      collection = ::Collection.find(collection_id) rescue nil
+      work = Object.const_get(ingest_work.work_type).new
+      #status_after, embargo_date, lease_date = nil, nil, nil
+      work.apply_depositor_metadata(user)
+      work.attributes = ingest_work.data
+      if ingest_work.data.has_key? 'downloadable'
+        # Convert string to boolean
+        work.downloadable = ActiveModel::Type::Boolean.new.cast(ingest_work.data['downloadable'])
+      elsif work.attributes.include? 'downloadable' # Set work to downloadable by default
+        work.downloadable = true
+      end
+      work.member_of_collections = [collection] if collection
+      work.admin_set = admin_set if admin_set
+      work.date_uploaded = DateTime.now
+      begin
+        work.save!
+      # Weird error where descriptions with whitespace chars \n or \r don't save the 1st time
+      # but do on the second
+      rescue Ldp::BadRequest
+        old_descr = work.description.clone.to_a
+        work.description = work.description.map { |w| w.gsub("\n","").gsub("\r","") }
+        work.save!
+        work.description = old_descr
+        work.save!
+      end
+      # To use the BatchCreateFilesWithOrderedMembersJob instead, replace the following line
+      # with BatchCreateFilesWithOrderedMembersJob.perform_later(work, ingest_work, user)
+      BatchCreateFilesJob.perform_later(work, ingest_work, user)
+    end
+  end
 end

data/app/jobs/cdm_migrator/restart_upload_from_middle_job.rb

@@ -0,0 +1,36 @@
+module CdmMigrator
+  class RestartUploadFromMiddleJob < ActiveJob::Base
+
+    # For restarting failed CdmMigrator::BatchCreateFilesJobs
+    # from the middle instead of from the beginning (default
+    # behaviour in Sidekiq). This prevents duplicating file
+    # sets within a work.
+    #
+    retry_on Net::OpenTimeout, attempts: 20
+    retry_on Errno::ECONNRESET, attempts: 20
+
+    def perform(work, ingest_work, user)
+      ordered_members = work.ordered_members.to_a
+      last_file_set = ordered_members.last
+      # If the last file set's file was never ingested, do that now
+      if last_file_set.present? and last_file_set.files.none?
+        CdmMigrator::CdmIngestFilesJob.perform_later(last_file_set, last_file_set.import_url, user)
+      end
+      files = ingest_work.files[ordered_members.length..-1]
+      files.each do |file|
+        url = file[:url]
+        last_file = ingest_work.files.last==file
+        ::FileSet.new(import_url: url, label: file[:title]) do |fs|
+          fs.save
+          actor = Hyrax::Actors::FileSetActor.new(fs, user)
+          actor.create_metadata
+          actor.attach_file_to_work(work)
+          fs.attributes = file[:metadata]
+          fs.save!
+          CdmMigrator::CdmIngestFilesJob.perform_later(fs, url, user, ingest_work, last_file)
+        end
+      end
+
+    end
+  end
+end

data/app/jobs/cdm_migrator/update_object_job.rb

@@ -1,10 +1,10 @@
-module CdmMigrator
-  class UpdateObjectJob < ActiveJob::Base
-
-
-    def perform(attributes)
-      obj = ActiveFedora::Base.find
-
-    end
-  end
-end
+module CdmMigrator
+  class UpdateObjectJob < ActiveJob::Base
+
+
+    def perform(attributes)
+      obj = ActiveFedora::Base.find
+
+    end
+  end
+end

data/app/mailers/cdm_migrator/application_mailer.rb

@@ -1,6 +1,6 @@
-module CdmMigrator
-  class ApplicationMailer < ActionMailer::Base
-    default from: 'from@example.com'
-    layout 'mailer'
-  end
-end
+module CdmMigrator
+  class ApplicationMailer < ActionMailer::Base
+    default from: 'from@example.com'
+    layout 'mailer'
+  end
+end

data/app/models/cdm_migrator/application_record.rb

@@ -1,5 +1,5 @@
-module CdmMigrator
-  class ApplicationRecord < ActiveRecord::Base
-    self.abstract_class = true
-  end
-end
+module CdmMigrator
+  class ApplicationRecord < ActiveRecord::Base
+    self.abstract_class = true
+  end
+end

data/app/models/cdm_migrator/batch_ingest.rb

@@ -1,33 +1,33 @@
-module CdmMigrator
-  class BatchIngest < ActiveRecord::Base
-    serialize :data
-
-    def name
-      csv.split('/').last.gsub(/[0-9]{10}/,"")
-    end
-
-    def progress
-      return "Complete" if complete?
-      completed = IngestWork.where(batch_ingest_id: id, complete: true ).length
-      if completed==data.length
-        complete=true
-        save
-        "Complete"
-      else
-        "#{completed.to_s}/#{size}"
-      end
-    end
-
-    def username
-      @username ||= User.find(user_id).name
-    end
-
-    def complete?
-      complete
-    end
-
-    def message?
-      not(message.nil?||message.empty?)
-    end
-  end
-end
+module CdmMigrator
+  class BatchIngest < ActiveRecord::Base
+    serialize :data
+
+    def name
+      csv.split('/').last.gsub(/[0-9]{10}/,"")
+    end
+
+    def progress
+      return "Complete" if complete?
+      completed = IngestWork.where(batch_ingest_id: id, complete: true ).length
+      if completed==data.length
+        complete=true
+        save
+        "Complete"
+      else
+        "#{completed.to_s}/#{size}"
+      end
+    end
+
+    def username
+      @username ||= User.find(user_id).name
+    end
+
+    def complete?
+      complete
+    end
+
+    def message?
+      not(message.nil?||message.empty?)
+    end
+  end
+end

data/app/models/cdm_migrator/ingest_work.rb

@@ -1,16 +1,7 @@
-module CdmMigrator
-  class IngestWork < ActiveRecord::Base
-
-    def initialize(work, ingest_id)
-      super({
-        work_type: work[:type],
-        data: work[:metadata],
-        files: work[:files],
-        batch_ingest_id: ingest_id
-            })
-    end
-
-    serialize :data
-    serialize :files
-  end
-end
+module CdmMigrator
+  class IngestWork < ApplicationRecord
+
+    serialize :data
+    serialize :files
+  end
+end

data/app/views/cdm_migrator/cdm/collection.html.erb

@@ -1,12 +1,12 @@
-<h1><span class="fa fa-map"></span> <%= default_page_title %></h1>
-
-<h5>ContentDM API address: <%= @cdm_url %></h5>
-<%= form_tag cdm_mappings_path, method: "post", multipart: true do %>
-  <table>
-    <tr><td>Collection: </td><td><%= select_tag 'collection', options_for_select(@collections) %></td></tr>
-    <tr><td>Work: </td><td><%= select_tag 'work', options_for_select(@available_concerns) %></td></tr>
-    <tr><td>Use Local File Storage: </td><td><%= check_box_tag 'file_system', 'true', false %></td></tr>
-    <tr><td>Template to Apply: </td><td><%= file_field_tag 'template' %> (.yml file)</td></tr>
-  </table>
-  <%= submit_tag 'choose mappings' %>
+<h1><span class="fa fa-map"></span> <%= default_page_title %></h1>
+
+<h5>ContentDM API address: <%= @cdm_url %></h5>
+<%= form_tag cdm_mappings_path, method: "post", multipart: true do %>
+  <table>
+    <tr><td>Collection: </td><td><%= select_tag 'collection', options_for_select(@collections) %></td></tr>
+    <tr><td>Work: </td><td><%= select_tag 'work', options_for_select(@available_concerns) %></td></tr>
+    <tr><td>Use Local File Storage: </td><td><%= check_box_tag 'file_system', 'true', false %></td></tr>
+    <tr><td>Template to Apply: </td><td><%= file_field_tag 'template' %> (.yml file)</td></tr>
+  </table>
+  <%= submit_tag 'choose mappings' %>
 <% end %>

data/app/views/cdm_migrator/cdm/mappings.html.erb

@@ -1,53 +1,54 @@
-<style type="text/css" media="screen">
-
-table{
-border-collapse:collapse;
-border:1px solid black;
-}
-
-table td{
-border:1px solid black;
-}
-</style>
-
-<h1><span class="fa fa-map"></span> <%= default_page_title %></h1>
-
-<%= form_tag cdm_generate_path.gsub("generate","generate.csv") do %>
-<%= hidden_field_tag "collection", params[:collection] %>
-<%= hidden_field_tag "file_system", params[:file_system] %>
-  <table>
-  <% @cdm_terms.each_with_index do |t,i| %>
-    <tr>
-    <td>
-    <%= label_tag t.first %>
-    <%= hidden_field_tag "mappings[#{i}][cdm]", t.last %>
-    </td>
-    <td>
-    <h3>Choose one</h3>
-    <%= label_tag 'applies to boths compound objects and children:' %><%= select_tag "mappings[#{i}][hydra]", options_for_select(@terms, @yaml.nil? ? "" : @yaml[t.last]['hydra']), include_blank: true %><br>
-    <%= label_tag 'applies only to compound objects: ' %><%= select_tag "mappings[#{i}][hydrac]", options_for_select(@work_only, @yaml.nil? ? "" : @yaml[t.last]['hydrac']), include_blank: true %>
-    </td>
-    </tr>
-  <% end %>
-  </table>
-  <%= select_tag "mappings_url", options_for_select(@dirs.sort { |x,y| x[0].downcase <=> y[0].downcase }) if @cdm_dirs %>
-  <%= hidden_field_tag "work", params[:work] %>
-  <%= submit_tag 'generate csv'%>
-<% end %>
-
-<%= button_tag "Download Template", { "onclick" => "template()" } %>
-
-<script type="text/javascript">
-  function template() {
-      document.getElementsByTagName("form")[0].action = "<%= cdm_template_path.gsub("template","template.yml") %>";
-      document.getElementsByName("commit")[0].click();
-      document.getElementsByTagName("form")[0].action = "<%= cdm_generate_path.gsub("generate","generate.csv") %>";
-  }
-
-  inputs = document.getElementsByTagName('option');
-  for (var i = 0; i < inputs.length; i++) {
-      if (inputs[i].hasAttribute('selected')) {
-          inputs[i].selected = true;
-      }
-  }
-</script>
+<style type="text/css" media="screen">
+
+table{
+border-collapse:collapse;
+border:1px solid black;
+}
+
+table td{
+border:1px solid black;
+}
+</style>
+
+<h1><span class="fa fa-map"></span> <%= default_page_title %></h1>
+
+<%= form_tag cdm_generate_path.gsub("generate","generate.csv") do %>
+<%= hidden_field_tag "collection", params[:collection] %>
+<%= hidden_field_tag "file_system", params[:file_system] %>
+  <table>
+  <% @cdm_terms.each_with_index do |t,i| %>
+    <tr>
+    <td>
+    <%= label_tag t.first %>
+    <%= hidden_field_tag "mappings[#{i}][cdm]", t.last %>
+    </td>
+    <td>
+    <h3>Choose one</h3>
+    <%= label_tag 'applies to boths compound objects and children:' %><%= select_tag "mappings[#{i}][hydra]", options_for_select(@terms, @yaml.nil? ? "" : @yaml[t.last]['hydra']), include_blank: true %><br>
+    <%= label_tag 'applies only to compound objects: ' %><%= select_tag "mappings[#{i}][hydrac]", options_for_select(@work_only, @yaml.nil? ? "" : @yaml[t.last]['hydrac']), include_blank: true %>
+    </td>
+    </tr>
+  <% end %>
+  </table>
+  <%= select_tag "mappings_url", options_for_select(@dirs.sort { |x,y| x[0].downcase <=> y[0].downcase }) if @cdm_dirs %>
+  <%= hidden_field_tag "work", params[:work] %>
+  <%= submit_tag 'generate csv'%>
+<% end %>
+
+<%= button_tag "Download Template", { "onclick" => "template()" } %>
+<%= link_to "Map Another Collection", cdm_start_path, class: 'btn btn-primary' %>
+
+<script type="text/javascript">
+  function template() {
+      document.getElementsByTagName("form")[0].action = "<%= cdm_template_path.gsub("template","template.yml") %>";
+      document.getElementsByName("commit")[0].click();
+      document.getElementsByTagName("form")[0].action = "<%= cdm_generate_path.gsub("generate","generate.csv") %>";
+  }
+
+  inputs = document.getElementsByTagName('option');
+  for (var i = 0; i < inputs.length; i++) {
+      if (inputs[i].hasAttribute('selected')) {
+          inputs[i].selected = true;
+      }
+  }
+</script>

data/app/views/cdm_migrator/csv/_batches_list.html.erb

@@ -1,4 +1,4 @@
-<% # container for all batches in index view -%>
-<div class="table-responsive" id="batches">
-  <%= render 'default_group', batches: @batches %>
-</div>
+<% # container for all batches in index view -%>
+<div class="table-responsive" id="batches">
+  <%= render 'default_group', batches: @batches %>
+</div>

data/app/views/cdm_migrator/csv/_default_group.html.erb

@@ -1,17 +1,17 @@
-<table class="table table-striped works-list">
-  <thead>
-  <tr>
-    <th><%= "Name" %></th>
-    <th><%= "Uploaded" %></th>
-    <th><%= "User" %></th>
-    <th><%= "Progress" %></th>
-    <th><%= "Message" %></th>
-    <th><%= "Actions" %></th>
-  </tr>
-  </thead>
-  <tbody>
-  <% batches.each do |batch| %>
-      <%= render 'list_batches', batch: batch %>
-  <% end %>
-  </tbody>
-</table>
+<table class="table table-striped works-list">
+  <thead>
+  <tr>
+    <th><%= "Name" %></th>
+    <th><%= "Uploaded" %></th>
+    <th><%= "User" %></th>
+    <th><%= "Progress" %></th>
+    <th><%= "Message" %></th>
+    <th><%= "Actions" %></th>
+  </tr>
+  </thead>
+  <tbody>
+  <% batches.each do |batch| %>
+      <%= render 'list_batches', batch: batch %>
+  <% end %>
+  </tbody>
+</table>