cdm_migrator 2.0.0 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e8c4fdc95bd2d038a441f126555e80767c40b05a7f9ed20eb1aae116f2a66921
-  data.tar.gz: 1a9a1695951cbb45f6ad78855223f62900a9033ea6b749db77827b67745b5b4f
+  metadata.gz: ce7a2c1babecf6e2c4da24d3afe33d9f16229418c8b18484cfa0ac546a45b9d6
+  data.tar.gz: fa52d0f174a46612fa1b42226b5be5c521b2b8c6da64ce03f59bb890fec3a981
 SHA512:
-  metadata.gz: f123901208397e46b758690c7de039712b0f3da5d4375e9ca8102668914994c698114f29ac964120fb69e0d88b0964b0e9a25a72dde6b1f6987826c024239e20
-  data.tar.gz: ca95b987182c67fdca1d21d3317658f8895b66ca74eb2eb8e57e94d501930799a700c532afc079cfc1509c00ce14ac3d3c9f9e757280d34bfa33e05b1f8bb36c
+  metadata.gz: 6ef368921d6f641c8493c4b7a9e61f414dcb802db7a80dcaf4d51ba335cc2bff62dc56f047442b7372e98fcce825d4bc9fe62ab180f8d38fd489ff1ae7841a2b
+  data.tar.gz: c48c112970bb36749528faf4e8a84382c50c5564b8a73c27dfbc2a7dccccb5fc5c29f570906e0ef7f108bf05ce322ecf508f20caeaa48ceb7a590958be4ffd6d

data/README.md

@@ -21,6 +21,7 @@ $ gem install cdm_migrator
 Finally, run:
 ```bash
 $ rails g cdm_migrator:install
+$ rails db:migrate
 ```
 to insert the yml and add a link to your Hyrax dashboard
 

data/app/controllers/cdm_migrator/cdm_controller.rb

@@ -1,7 +1,7 @@
 module CdmMigrator
 	class CdmController < ApplicationController
 		helper_method :default_page_title, :admin_host?, :available_translations, :available_works
-		layout 'dashboard' if Hyrax
+		layout 'hyrax/dashboard' if Hyrax
 		require 'csv'
 
 		before_action :load_yaml
@@ -29,7 +29,7 @@ module CdmMigrator
 				records = []
 				[0..(total_recs/1024)].each do |index|
 					start = (index*1024) + 1
-					json = JSON.parse(Net::HTTP.get_response(URI.parse("http://#{@cdm_url}:#{@cdm_port}/dmwebservices/index.php?q=dmQuery#{params[:collection]}/0/0/filetype/1024/#{start}/0/0/0/0/1/0/json")).body)
+					json = JSON.parse(Net::HTTP.get_response(URI.parse("#{@cdm_url}:#{@cdm_port}/dmwebservices/index.php?q=dmQuery#{params[:collection]}/0/0/filetype/1024/#{start}/0/0/0/0/1/0/json")).body)
 					records << json['records'].map { |rec| [rec['pointer'], rec['filetype']] }
 				end
 			else

data/app/controllers/cdm_migrator/csv_controller.rb

@@ -1,175 +1,184 @@
 module CdmMigrator
 	class CsvController < ApplicationController
 		helper_method :default_page_title, :admin_host?, :available_translations, :available_works
-		layout 'dashboard'
-		
-		def generate
-			headers = ['type','url']
-			skip = ["id", "head", "tail", "depositor", "date_uploaded", "date_modified", "import_url", "thumbnail_id", "embargo_id", "lease_id", "access_control_id", "representative_id"]
-			GenericWork.new.attributes.each do |key, val|
-				headers << "work_#{key}" unless skip.include? key
-			end
-			FileSet.new.attributes.each do |key, val|
-				headers << "file_#{key}" unless skip.include? key
-			end
-			fname = "template_#{DateTime.now.to_i}"
-			render plain: CSV.generate { |csv| csv << headers }, content_type: 'text/csv'
-		end
+    include ActionView::Helpers::UrlHelper
+		layout 'hyrax/dashboard' if Hyrax
+    before_action :authenticate, except: :index
+
+    def file_path_checker
+     if params[:file]
+       check_paths params[:file].path
+
+       if @path_list.blank?
+         flash[:notice] = "All file paths are valid."
+       else
+         flash[:error] = "Cdm Migrator couldn't find files at the following urls. Please correct the paths and try again."
+       end
+     end
+    end
+
+    def index
+      if current_page?(main_app.csv_my_batches_path(locale: nil))
+        @batches = BatchIngest.where(user_id: current_user.id).reverse_order
+      elsif current_page?(main_app.csv_all_batches_path(locale: nil))
+        @batches = BatchIngest.all.reverse_order
+      else
+        @batches = []
+      end
+    end
 
 		def upload
-			#byebug
-			authorize! :create, available_works.first
 			@admin_sets = AdminSet.all.map { |as| [as.title.first, as.id] }
 			@collections = Collection.all.map { |col| [col.title.first, col.id] }
 		end
 
 		def create
-			#byebug
-			authorize! :create, available_works.first
 			dir = Rails.root.join('public', 'uploads', 'csvs')
-			Dir.mkdir(dir) unless Dir.exist?(dir)
-			File.open(dir.join(params[:csv_import][:csv_file].original_filename), 'wb') do |file|
+			FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
+			time = DateTime.now.strftime('%s')
+			filename = params[:csv_import][:csv_file].original_filename.gsub('.csv',"#{time}.csv")
+      csv = dir.join(filename).to_s
+			File.open(csv, 'wb') do |file|
 			  file.write(params[:csv_import][:csv_file].read)
-			end
-			csv = CSV.parse(File.read(dir.join(params[:csv_import][:csv_file].original_filename)), headers: true, encoding: 'utf-8')
-			CsvUploadJob.perform_later(dir.join(params[:csv_import][:csv_file].original_filename).to_s, params[:csv_import][:mvs], params[:collection], params[:admin_set], current_user)
-			#perform(params[:csv_import][:csv_file].path, params[:csv_import][:mvs], current_user)
-			flash[:notice] = "csv successfully uploaded"
-			redirect_to csv_upload_path
-		end
+      end
+      check_paths csv
+      if @path_list.present?
+        flash[:error] = "some file paths are invalid"
+      end
+      parse_csv(csv, params[:csv_import][:mvs])
+
+      ingest = BatchIngest.new({
+                                   data: @works,
+                                   size: @works.length,
+                                   csv: csv,
+                                   admin_set_id: params[:admin_set],
+                                   collection_id: params[:collection],
+                                   user_id: current_user.id,
+                                   message: @path_list.blank? ? nil : @path_list.to_s.gsub("\"","&quot;")
+                               })
+      if ingest.save! && @path_list.blank?
+        BatchCreateWorksJob.perform_later(ingest, current_user)
+        flash[:notice] = "csv successfully uploaded, check this page to see the status while the batch is running"
+        redirect_to csv_my_batches_path
+      else
+        flash[:error] ||= "csv could not be parsed, please check and re-upload"
+        redirect_to csv_upload_path
+      end
+    end
+
+    def rerun
+      ingest = BatchIngest.find(params[:id]).deep_dup
+      ingest.save
+      BatchCreateWorksJob.perform_later(ingest, current_user)
+      flash[:notice] = "csv successfully uploaded, check this page to see the status while the batch is running"
+      redirect_to csv_my_batches_path
+    end
+
+    def generate
+      headers = ['type','url']
+      skip = ["id", "head", "tail", "depositor", "date_uploaded", "date_modified", "import_url", "thumbnail_id",
+              "embargo_id", "lease_id", "access_control_id", "representative_id"]
+      GenericWork.new.attributes.each do |key, val|
+        headers << "work_#{key}" unless skip.include? key
+      end
+      FileSet.new.attributes.each do |key, val|
+        headers << "file_#{key}" unless skip.include? key
+      end
+      fname = "template_#{DateTime.now.to_i}"
+      render plain: CSV.generate { |csv| csv << headers }, content_type: 'text/csv'
+    end
 		
 		private
 
-		def perform(csv, mvs, current_user)
-			@csv = CSV.parse(File.read(csv), headers: true, encoding: 'utf-8').map(&:to_hash)
-			@mvs = mvs
-			@collection = Collection.find(params[:csv_import][:collection]) rescue nil
-			@admin_set = AdminSet.find(params[:csv_import][:admin_set]) rescue nil
-			@works = []
-			@files = {}
-			@csv.each do |row|
-				type = row.first.last
-				if type.nil?
-					next
-				elsif(type.include? "Work")
-					@works << row
-					@files[@works.length] = []
-				elsif(type.include? "File")
-					row.delete("object_type")
-					@files[@works.length] << row
-				end
-			end
-			create_works
-		end
-
-		def available_works
-			@available_works ||= Hyrax::QuickClassificationQuery.new(current_user).authorized_models
-		end
-		
-		def default_page_title
-			'CSV Batch Uploader'
-		end
-		
-		def admin_host?
-			false unless Settings.multitenancy.enabled
-		end
-		
-		def available_translations
-			{
-				'en' => 'English',
-				'fr' => 'French'
-			}
-		end
-
-		def work_form
-			Module.const_get("Hyrax::#{params[:work]}Form") rescue nil || Module.const_get("Hyrax::Forms::WorkForm")
-		end
-
-		def file_form
-			Module.const_get("Hyrax::FileSetForm") rescue nil || Module.const_get("Hyrax::Forms::FileSetEditForm")
-		end
-
-			def secondary_terms form_name
-				form_name.terms - form_name.required_fields -
-						[:visibility_during_embargo, :embargo_release_date,
-						 :visibility_after_embargo, :visibility_during_lease,
-						 :lease_expiration_date, :visibility_after_lease, :visibility,
-						 :thumbnail_id, :representative_id, :ordered_member_ids,
-						 :collection_ids, :in_works_ids, :admin_set_id, :files, :source, :member_of_collection_ids]
-			end
-
-			def create_file_from_url(url, file_name, work, file_data)
-				::FileSet.new(import_url: url, label: file_name) do |fs|
-					fs.save
-					actor = Hyrax::Actors::FileSetActor.new(fs, current_user)
-					actor.create_metadata#(work, visibility: work.visibility)
-					actor.attach_file_to_work(work)
-					#byebug
-					fs.attributes = file_data
-					fs.save!
-					uri = URI.parse(url.gsub(' ','%20'))
-					if uri.scheme == 'file'
-						IngestLocalFileJob.perform_later(fs, uri.path.gsub('%20',' '), current_user)
-					else
-						ImportUrlJob.perform_later(fs, log(actor.user))
-					end
-				end
-			end
-			#
-			def load_metadata(fs, file_array)
-				file_array.each do |line|
-					fileset = fs
-					index = -1
-					line.each do |data|
-						index = index + 1
-						next if index==0
-						if @csv.headers[index] == "visibility"
-							fileset.visibility = data
-						elsif @csv.headers[index] == "depositor"
-							fileset.depositor = data
-						else
-							data_arr = data.split @mvs
-							fileset[@csv.headers[index]] = data_arr
-						end
-					end
-					fileset.save
-				end
-			end
-
-			def create_works
-				index = 1
-				@works.each do |work_data|
-					work = Object.const_get(work_data.first.last).new#delete("object_type")).new
-					status_after, embargo_date, lease_date = nil, nil, nil
-					final_work_data = create_data work_data, work_form, work
-					work.apply_depositor_metadata(current_user)
-					work.attributes = final_work_data
-					work.member_of_collections = [@collection] if @collection
-					work.admin_set = @admin_set if @admin_set
-					work.save
-					create_files(work, index)
-					index+=1
-				end
-			end
-
-			def create_data data, type, object
-				final_data = {}
-				accepted_terms = type.required_fields + secondary_terms(type)
-				data.each do |key, att|
-					if(att.nil? || att.empty? || key.to_s.include?("object_type") || !accepted_terms.include?(key.to_sym) )
-						next
-					elsif(object.send(key).nil?)
-						final_data[key] = att
-					else
-						final_data[key] = att.split @mvs
-					end
-				end
-				final_data
-			end
+      def authenticate
+        authorize! :create, available_works.first
+      end
+
+      def available_works
+        @available_works ||= Hyrax::QuickClassificationQuery.new(current_user).authorized_models
+      end
+
+      def parse_csv csv, mvs
+        csv = CSV.parse(File.read(csv), headers: true, encoding: 'utf-8').map(&:to_hash)
+        @works = []
+        csv.each do |row|
+          type = row.first.last
+          if type.nil?
+            next
+          elsif type.include? "Work"
+            metadata = create_data(row, work_form(type), Object.const_get(type).new, mvs)
+            @works << {type: type, metadata: metadata, files: []}
+          elsif type.include? "File"
+            metadata = create_data(row, file_form, FileSet.new, mvs)
+            @works.last[:files] << {url: row.delete('url'), title: row.delete('title'), metadata: metadata}
+          end
+        end
+      end
+
+      def check_paths csv_file
+        row_number = 1 # +1 offset to account for csv headers
+        @path_list = {}
+
+        CSV.foreach(csv_file, headers: true, header_converters: :symbol) do |row|
+          row_number +=1 # Tells user what CSV row the bogus file path is on
+          next if row[:url].nil?
+          file_path = row[:url]
+          unless File.file?(file_path.gsub("file://", ""))
+            @path_list[row_number] = file_path
+          end
+        end
+      end
+
+      def default_page_title
+        'CSV Batch Uploader'
+      end
+
+      def admin_host?
+        false unless Settings.multitenancy.enabled
+      end
+
+      def available_translations
+        {
+          'en' => 'English',
+          'fr' => 'French'
+        }
+      end
+
+      def work_form(worktype = "GenericWork")
+        Module.const_get("Hyrax::#{worktype}Form") rescue nil || Module.const_get("Hyrax::Forms::WorkForm")
+      end
+
+      def file_form
+        Module.const_get("Hyrax::FileSetForm") rescue nil || Module.const_get("Hyrax::Forms::FileSetEditForm")
+      end
+
+      def secondary_terms form_name
+        form_name.terms - form_name.required_fields -
+            [:visibility_during_embargo, :embargo_release_date,
+             :visibility_after_embargo, :visibility_during_lease,
+             :lease_expiration_date, :visibility_after_lease, :visibility,
+             :thumbnail_id, :representative_id, :ordered_member_ids,
+             :collection_ids, :in_works_ids, :admin_set_id, :files, :source, :member_of_collection_ids]
+      end
+
+      def create_data data, type, object, mvs
+        final_data = {}
+        accepted_terms = type.required_fields + secondary_terms(type)
+        data.each do |key, att|
+          if(att.nil? || att.empty? || key.to_s.include?("object_type") || !accepted_terms.include?(key.to_sym) )
+            next
+          elsif(object.send(key).nil?)
+            final_data[key] = att
+          else
+            final_data[key] = att.split(mvs)
+          end
+        end
+        final_data
+      end
 
 			def create_lease visibility, status_after, date
 				lease = Hydra::AccessControls::Lease.new(visibility_during_lease: visibility,
-							visibility_after_lease: @status_after, lease_expiration_date: @lease_date)
+							visibility_after_lease: status_after, lease_expiration_date: @lease_date)
 				lease.save
 			end
 
@@ -181,26 +190,9 @@ module CdmMigrator
 				embargo.save
 			end
 
-			def create_files(work, index)
-				file = FileSet.new
-				@files[index].each do |file_data|
-					url = file_data.delete('url')
-					title = file_data.delete('title')
-					final_file_data = create_data file_data, file_form, file
-					create_file_from_url(url, title, work, final_file_data)
-				end
-			end
-
-	#		def log(user)
-	#			CurationConcerns::Operation.create!(user: user,
-	#												operation_type: "Attach Remote File")
-	#		end
-
-
-
-		def log(user)
-				Hyrax::Operation.create!(user: user,
-																						operation_type: "Attach Remote File")
-		end
+      def log(user)
+          Hyrax::Operation.create!(user: user,
+                                              operation_type: "Attach Remote File")
+      end
 	end
 end

data/app/jobs/cdm_migrator/batch_create_files_job.rb

@@ -0,0 +1,22 @@
+module CdmMigrator
+  class BatchCreateFilesJob < ActiveJob::Base
+    queue_as Hyrax.config.ingest_queue_name
+
+    def perform work, ingest_work, user, last_work=false
+      ingest_work.files.each do |file|
+        url = file[:url]
+        last_file = ingest_work.files.last==file
+        ::FileSet.new(import_url: url, label: file[:title]) do |fs|
+          fs.save
+          actor = Hyrax::Actors::FileSetActor.new(fs, user)
+          actor.create_metadata#(work, visibility: work.visibility)
+          actor.attach_file_to_work(work)
+          #byebug
+          fs.attributes = file[:metadata]
+          fs.save!
+          CdmIngestFilesJob.perform_later(fs, url, user, ingest_work, last_file, last_work)
+        end
+      end
+    end
+  end
+end

data/app/jobs/cdm_migrator/batch_create_works_job.rb

@@ -0,0 +1,15 @@
+module CdmMigrator
+  class BatchCreateWorksJob < ActiveJob::Base
+    queue_as Hyrax.config.ingest_queue_name
+
+    def perform(ingest, user)
+      ingest.data.each do |w|
+        last_work = ingest.data.last==w
+        ingest_work = IngestWork.new(w, ingest.id)
+        ingest_work.save!
+        CreateWorkJob.perform_later ingest_work, user, ingest.admin_set_id, ingest.collection_id, last_work
+      end
+
+    end
+  end
+end

data/app/jobs/cdm_migrator/cdm_ingest_files_job.rb

@@ -0,0 +1,35 @@
+module CdmMigrator
+  class CdmIngestFilesJob < ActiveJob::Base
+    queue_as Hyrax.config.ingest_queue_name
+
+    def perform(fs, url, user, ingest_work = nil, last_file = false, last_work = false)
+      if url.include?("http") && File.extname(url).include?("pdf")
+        download = open(url)
+        dir = Rails.root.join('public', 'uploads', 'csv_pdfs')
+        FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
+        filename = download.base_uri.to_s.split('/').last
+        url = dir.join(filename)
+        if fs.title.empty?
+          fs.title << filename.split('.').first
+          fs.save
+        end
+        IO.copy_stream(download, url)
+        url = "file://"+url.to_s
+      end
+      uri = URI.parse(url.gsub(' ','%20'))
+      if uri.scheme == 'file'
+        IngestLocalFileJob.perform_now(fs, uri.path.gsub('%20',' '), user)
+      else
+        ImportUrlJob.perform_now(fs, log(user))
+      end
+      ingest_work.update_attribute('complete', true) if last_file
+      BatchIngest.find(ingest_work.id).update_attribute('complete', true) if last_work
+    end
+
+    def log(user)
+      Hyrax::Operation.create!(user: user,
+                               operation_type: "Attach Remote File")
+    end
+
+  end
+end

data/app/jobs/cdm_migrator/create_work_job.rb

@@ -0,0 +1,20 @@
+module CdmMigrator
+  class CreateWorkJob < ActiveJob::Base
+    queue_as Hyrax.config.ingest_queue_name
+
+    def perform(ingest_work, user, admin_set_id, collection_id, last_work=false)
+      admin_set = ::AdminSet.find(admin_set_id) rescue nil
+      collection = ::Collection.find(collection_id) rescue nil
+      work = Object.const_get(ingest_work.work_type).new
+      #status_after, embargo_date, lease_date = nil, nil, nil
+      work.apply_depositor_metadata(user)
+      work.attributes = ingest_work.data
+      work.member_of_collections = [collection] if collection
+      work.admin_set = admin_set if admin_set
+      work.date_uploaded = DateTime.now
+      work.save
+      BatchCreateFilesJob.perform_later work, ingest_work, user, last_work
+
+    end
+  end
+end

data/app/models/cdm_migrator/batch_ingest.rb

@@ -0,0 +1,30 @@
+module CdmMigrator
+  class BatchIngest < ActiveRecord::Base
+    serialize :data
+
+    def name
+      csv.split('/').last.gsub(/[0-9]{10}/,"")
+    end
+
+    def progress
+      if complete?
+        "Complete"
+      else
+        completed = IngestWork.where(batch_ingest_id: id, complete: true ).length.to_s
+        "#{completed}/#{size}"
+      end
+    end
+
+    def username
+      @username ||= User.find(user_id).name
+    end
+
+    def complete?
+      self.complete
+    end
+
+    def message?
+      not(message.nil?||message.empty?)
+    end
+  end
+end

data/app/models/cdm_migrator/ingest_work.rb

@@ -0,0 +1,16 @@
+module CdmMigrator
+  class IngestWork < ActiveRecord::Base
+
+    def initialize(work, ingest_id)
+      super({
+        work_type: work[:type],
+        data: work[:metadata],
+        files: work[:files],
+        batch_ingest_id: ingest_id
+            })
+    end
+
+    serialize :data
+    serialize :files
+  end
+end

data/app/views/cdm_migrator/csv/_batches_list.html.erb

@@ -0,0 +1,4 @@
+<% # container for all batches in index view -%>
+<div class="table-responsive" id="batches">
+  <%= render 'default_group', batches: @batches %>
+</div>

data/app/views/cdm_migrator/csv/_default_group.html.erb

@@ -0,0 +1,17 @@
+<table class="table table-striped works-list">
+  <thead>
+  <tr>
+    <th><%= "Name" %></th>
+    <th><%= "Uploaded" %></th>
+    <th><%= "User" %></th>
+    <th><%= "Progress" %></th>
+    <th><%= "Message" %></th>
+    <th><%= "Actions" %></th>
+  </tr>
+  </thead>
+  <tbody>
+  <% batches.each do |batch| %>
+      <%= render 'list_batches', batch: batch %>
+  <% end %>
+  </tbody>
+</table>

data/app/views/cdm_migrator/csv/_list_batches.html.erb

@@ -0,0 +1,21 @@
+<tr id="batch_<%= batch.id %>">
+
+  <td><%= batch.name %></td>
+
+  <td class="date"><%= batch.created_at %></td>
+
+  <td><%= batch.username %></td>
+
+
+  <td><%= batch.progress %></td>
+  <td><%= batch.message? ? "<button onclick=\"alert('#{batch.message.html_safe}');\">View Message</button>".html_safe : "No Message" %></td>
+
+  <td>
+    <%= link_to(
+            "Run Again",
+            main_app.csv_rerun_path(id: batch.id),
+            data: { 'create-type' => 'batch' },
+            class: 'btn btn-primary'
+        ) if batch.complete? && (current_ability.admin? || current_page?(main_app.csv_my_batches_path(locale: nil))) %>
+  </td>
+</tr>

data/app/views/cdm_migrator/csv/_path_list.html.erb

@@ -0,0 +1,19 @@
+<% if @path_list && @path_list.any? %>
+    <table class="table table-striped">
+      <thead>
+        <tr>
+          <th scope="col" style="min-width: 100px;">Line No.</th>
+          <th scope="col">File Path (url)</th>
+        </tr>
+      </thead>
+      <tbody>
+        <% @path_list.each do |line, path| %>
+          <tr>
+            <td><%= line %></td>
+            <td><%= path %></td>
+          </tr>
+        <% end %>
+      </tbody>
+    </table>
+<% end %>
+

data/app/views/cdm_migrator/csv/_results_pagination.html.erb

@@ -0,0 +1,9 @@
+<% if @response.total_pages > 1 %>
+ <div class="row record-padding">
+  <div class="col-md-9">
+    <div class="pagination">
+      <%= paginate @response, outer_window: 2, theme: 'blacklight', route_set: hyrax %>
+    </div>
+  </div>
+ </div>
+<% end %>

data/app/views/cdm_migrator/csv/_tabs.html.erb

@@ -0,0 +1,8 @@
+<ul class="nav nav-tabs" id="my_nav" role="navigation">
+  <li<%= ' class="active"'.html_safe if current_page?(main_app.csv_all_batches_path(locale: nil)) %>>
+    <%= link_to "All CSV Batches", main_app.csv_all_batches_path %>
+  </li>
+  <li<%= ' class="active"'.html_safe if current_page?(main_app.csv_my_batches_path(locale: nil)) %>>
+    <%= link_to "My CSV Batches", main_app.csv_my_batches_path %>
+  </li>
+</ul>

data/app/views/cdm_migrator/csv/file_path_checker.html.erb

@@ -0,0 +1,21 @@
+<% provide :page_title, "File Path Checker" %>
+
+<h1>File Path Checker</h1>
+
+<!-- <div class="row"> -->
+<p>This tool checks if a file exists at each url in a csv.</p>
+<%= form_tag(check_file_paths_path, remote: true, method: :post, multipart: true, id: "csv-form") do %>
+<div class="input-group">
+  <%= file_field_tag(:file, class: "form-control-file") %>
+  <%= hidden_field_tag :authenticity_token, value: form_authenticity_token %>
+  <%= button_tag(type: :submit, class: "btn btn-large btn-primary", style: "margin-top: 1em;") do %>
+    Check CSV
+  <% end %>
+<% end %>
+</div>
+<!-- </div> -->
+
+<div id="path_list" class="col-md-10 offset-md-1">
+  <%= render 'path_list' %>
+</div>
+

data/app/views/cdm_migrator/csv/index.html.erb

@@ -0,0 +1,19 @@
+<% provide :page_title, "Batches" %>
+
+
+<% provide :page_header do %>
+  <h1><span class="fa fa-database" aria-hidden="true"></span> <%= "Batches" %></h1>
+<% end %>
+
+<div class="row">
+  <div class="col-md-12">
+    <div class="panel panel-default tabs">
+      <%= render 'tabs' %>
+      <div class="panel-body">
+        <%= render 'batches_list' %>
+
+        <%#= render 'results_pagination' %>
+      </div>
+    </div>
+  </div>
+</div>

data/config/routes.rb

@@ -2,6 +2,12 @@ Rails.application.routes.draw do
   get '/cdm_migrator/upload', to: 'cdm_migrator/csv#upload', as: 'csv_upload'
   post '/cdm_migrator/upload', to: 'cdm_migrator/csv#create', as: 'csv_create'
   get '/cdm_migrator/generate', to: 'cdm_migrator/csv#generate', as: 'csv_generate'
+  get '/cdm_migrator/my/batches', to: 'cdm_migrator/csv#index', as: 'csv_my_batches'
+  get '/cdm_migrator/batches', to: 'cdm_migrator/csv#index', as: 'csv_all_batches'
+  get '/cdm_migrator/rerun/:id', to: 'cdm_migrator/csv#rerun', as: 'csv_rerun'
+
+  get '/cdm_migrator/file_path_checker', to: 'cdm_migrator/csv#file_path_checker', as: 'file_path_checker'
+  post '/cdm_migrator/file_path_checker', to: 'cdm_migrator/csv#file_path_checker', as: 'check_file_paths'
 
   get '/cdm_migrator/collection', to: 'cdm_migrator/cdm#collection', as: 'cdm_start'
   post '/cdm_migrator/mappings/', to: 'cdm_migrator/cdm#mappings', as: 'cdm_mappings'

data/db/migrate/20191211193859_create_batch_ingests.rb

@@ -0,0 +1,20 @@
+class CreateBatchIngests < ActiveRecord::Migration[5.0]
+  def up
+    create_table :batch_ingests do |t|
+      t.text :data
+      t.string :admin_set_id
+      t.string :collection_id
+      t.text :message
+      t.integer :size
+      t.string :csv
+      t.references :user, foreign_key: true
+      t.boolean :complete, default: false
+
+      t.timestamps
+    end
+  end
+
+  def down
+    drop_table :batch_ingests
+  end
+end

data/db/migrate/20191212192315_create_ingest_works.rb

@@ -0,0 +1,18 @@
+class CreateIngestWorks < ActiveRecord::Migration[5.0]
+  def up
+    create_table :ingest_works do |t|
+      t.string :work_type
+      t.text :data
+      t.text :files
+      t.boolean :complete, :default => false
+
+      t.references :batch_ingest, foreign_key: true
+
+      t.timestamps
+    end
+  end
+
+  def down
+    drop_table :ingest_works
+  end
+end

data/lib/cdm_migrator/engine.rb

@@ -1,15 +1,22 @@
-
-
 module CdmMigrator
   class Engine < ::Rails::Engine
 
+    initializer :append_migrations do |app|
+      unless app.root.to_s.match root.to_s
+        config.paths["db/migrate"].expanded.each do |expanded_path|
+          app.config.paths["db/migrate"] << expanded_path
+        end
+      end
+    end
+
     #isolate_namespace CdmMigrator
     class << self
-      
+
       def config
-        file = File.open(File.join(::Rails.root, "/config/cdm_migrator.yml"))
+        file    = File.open(File.join(::Rails.root, "/config/cdm_migrator.yml"))
         @config ||= YAML.safe_load(file)
       end
+
       # loads a yml file with the configuration options
       #
       # @param file [String] path to the yml file

data/lib/cdm_migrator/version.rb

@@ -1,3 +1,3 @@
 module CdmMigrator
-  VERSION = '2.0.0'
+  VERSION = '3.0.0'
 end

data/lib/generators/cdm_migrator/install/install_generator.rb

@@ -4,21 +4,36 @@ class CdmMigrator::InstallGenerator < Rails::Generators::Base
   def inject_dashboard_link
     file_path = "app/views/hyrax/dashboard/sidebar/_tasks.html.erb"
     if File.file?(file_path)
-      gsub_file file_path,/[ \t]*(<% if can\? :review, :submissions %>)\n[ \t]*(<li class="h5"><%= t\('hyrax\.admin\.sidebar\.tasks'\) %><\/li>)\n/ do |match|
-match.split("\n")[1].to_s+
-"\n	<li>\n" \
-"		<%= menu.collapsable_section t('CDM Migrator'),\n" \
-"							 icon_class: \"fa fa-map-signs\",\n" \
-"							 id: 'collapseCdmMigrator',\n" \
-"							 open: menu.cdm_migrator_section? do %>\n" \
-"			<%= menu.nav_link(main_app.csv_upload_path) do %>\n"\
-"				<span class=\"fa fa-angle-double-up\"></span> <span class=\"sidebar-action-text\"><%= t('CSV Batch Uploader') %></span>\n" \
-"			<% end %>\n" \
-"			<%= menu.nav_link(main_app.cdm_start_path) do %>\n" \
-"				<span class=\"fa fa-map\"></span> <span class=\"sidebar-action-text\"><%= t('CDM Mapping Tool') %></span>\n" \
-"			<% end %>\n" \
-"		<% end %>\n" \
-"	</li>\n" + match.split("\n")[0].to_s + "\n"
+      title = "<li class=\"h5\"><%= t('hyrax.admin.sidebar.tasks') %></li>"
+      perm = "  <% if can? :review, :submissions %>"
+      gsub_file file_path,/[ \t]*(<li class="h5"><%= t\('hyrax\.admin\.sidebar\.tasks'\) %><\/li>)\n[\s\S]*[ \t]*(<% if can\? :review, :submissions %>)\n/ do |match|
+        ""
+      end
+      gsub_file file_path,/[ \t]*(<% if can\? :review, :submissions %>)\n[\s\S]*[ \t]*(<li class="h5"><%= t\('hyrax\.admin\.sidebar\.tasks'\) %><\/li>)\n/ do |match|
+        ""
+      end
+      prepend_to_file file_path do
+            title + "\n" \
+            "<li>\n" \
+            "		<%= menu.collapsable_section t('CDM Migrator'),\n" \
+            "							 icon_class: \"fa fa-map-signs\",\n" \
+            "							 id: 'collapseCdmMigrator',\n" \
+            "							 open: menu.cdm_migrator_section? do %>\n" \
+            "			<%= menu.nav_link(main_app.cdm_start_path) do %>\n" \
+            "				<span class=\"fa fa-map\"></span> <span class=\"sidebar-action-text\"><%= t('CDM Mapping Tool') %></span>\n" \
+            "			<% end %>\n" \
+            "     <%= menu.nav_link(main_app.file_path_checker_path) do %>\n" \
+            "       <span class=\"fa fa-check-circle\"></span><span>File Path Checker</span>\n" \
+            "     <% end %>\n" \
+            "			<%= menu.nav_link(main_app.csv_upload_path) do %>\n"\
+            "				<span class=\"fa fa-angle-double-up\"></span> <span class=\"sidebar-action-text\"><%= t('CSV Batch Uploader') %></span>\n" \
+            "			<% end %>\n" \
+            "			<%= menu.nav_link(main_app.csv_my_batches_path) do %>\n" \
+            "				<span class=\"fa fa-database\"></span> <span class=\"sidebar-action-text\"><%= t('Batches') %></span>\n" \
+            "			<% end %>\n" \
+            "		<% end %>\n" \
+            "	</li>\n" + perm + "\n"
+
       end
     else
     	copy_file "sidebar/_tasks.html.erb", "app/views/hyrax/dashboard/sidebar/_tasks.html.erb"
@@ -28,14 +43,14 @@ match.split("\n")[1].to_s+
   def inject_menu_presenter
   	  hyku_file_path = "app/presenters/hyku/menu_presenter.rb"
   	  hyrax_file_path = "app/presenters/hyrax/menu_presenter.rb"
-  	  if File.file?(hyku_file_path)
+  	  if File.file?(hyku_file_path) && File.readlines(hyku_file_path).join.include?("cdm_migrator_section")
   	  	  insert_into_file hyku_file_path, :after => /def settings_section\?\n.*\(controller_name\)\n[ \t]*end/ do
   	  	  	  "\n\n" \
   	  	  	  "    def cdm_migrator_section?\n" \
   	  	  	  "      %w[cdm csv].include?(controller_name)\n" \
   	  	  	  "    end\n"
   	  	  end
-  	  elsif File.file?(hyrax_file_path)
+  	  elsif File.file?(hyrax_file_path) && File.readlines(hyrax_file_path).join.include?("cdm_migrator_section")
   	  	  insert_into_file hyrax_file_path, :after => /def settings_section\?\n.*\(controller_name\)\n[ \t]*end/ do
   	  	  	  "\n\n" \
   	  	  	  "    def cdm_migrator_section?\n" \
@@ -50,7 +65,7 @@ match.split("\n")[1].to_s+
   end
 
   def inject_content_dm_yml
-    copy_file "config/cdm_migrator.yml", "config/cdm_migrator.yml"
+    copy_file("config/cdm_migrator.yml", "config/cdm_migrator.yml") unless File.file?("config/cdm_migrator.yml")
   end
   
-end
+end

data/lib/generators/cdm_migrator/install/templates/sidebar/_tasks.html.erb

@@ -4,12 +4,20 @@
 							 icon_class: "fa fa-map-signs",
 							 id: 'collapseCdmMigrator',
 							 open: menu.cdm_migrator_section? do %>
+			<%= menu.nav_link(main_app.cdm_start_path) do %>
+				<span class="fa fa-map"></span> <span class="sidebar-action-text"><%= t('CDM Mapping Tool') %></span>
+			<% end %>
+
+      <%= menu.nav_link(main_app.file_path_checker_path) do %>
+        <span class="fa fa-check-circle"></span><span class="sidebar-action-text"><%= t('File Path Checker') %></span>
+      <% end %>
+
 			<%= menu.nav_link(main_app.csv_upload_path) do %>
 				<span class="fa fa-angle-double-up"></span> <span class="sidebar-action-text"><%= t('CSV Batch Uploader') %></span>
 			<% end %>
 			
-			<%= menu.nav_link(main_app.cdm_start_path) do %>
-				<span class="fa fa-map"></span> <span class="sidebar-action-text"><%= t('CDM Mapping Tool') %></span>
+			<%= menu.nav_link(main_app.csv_my_batches_path) do %>
+				<span class="fa fa-database"></span> <span class="sidebar-action-text"><%= t('Batches') %></span>
 			<% end %>
 		<% end %>
 	</li>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cdm_migrator
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 3.0.0
 platform: ruby
 authors:
 - sephirothkod
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-01-29 00:00:00.000000000 Z
+date: 2019-12-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -86,14 +86,29 @@ files:
 - app/controllers/cdm_migrator/csv_controller.rb
 - app/helpers/cdm_migrator/application_helper.rb
 - app/jobs/cdm_migrator/application_job.rb
-- app/jobs/csv_upload_job.rb
+- app/jobs/cdm_migrator/batch_create_files_job.rb
+- app/jobs/cdm_migrator/batch_create_works_job.rb
+- app/jobs/cdm_migrator/cdm_ingest_files_job.rb
+- app/jobs/cdm_migrator/create_work_job.rb
 - app/mailers/cdm_migrator/application_mailer.rb
 - app/models/cdm_migrator/application_record.rb
+- app/models/cdm_migrator/batch_ingest.rb
+- app/models/cdm_migrator/ingest_work.rb
 - app/views/cdm_migrator/cdm/collection.html.erb
 - app/views/cdm_migrator/cdm/mappings.html.erb
+- app/views/cdm_migrator/csv/_batches_list.html.erb
+- app/views/cdm_migrator/csv/_default_group.html.erb
+- app/views/cdm_migrator/csv/_list_batches.html.erb
+- app/views/cdm_migrator/csv/_path_list.html.erb
+- app/views/cdm_migrator/csv/_results_pagination.html.erb
+- app/views/cdm_migrator/csv/_tabs.html.erb
+- app/views/cdm_migrator/csv/file_path_checker.html.erb
+- app/views/cdm_migrator/csv/index.html.erb
 - app/views/cdm_migrator/csv/upload.html.erb
 - app/views/layouts/cdm_migrator/application.html.erb
 - config/routes.rb
+- db/migrate/20191211193859_create_batch_ingests.rb
+- db/migrate/20191212192315_create_ingest_works.rb
 - lib/cdm_migrator.rb
 - lib/cdm_migrator/engine.rb
 - lib/cdm_migrator/version.rb

data/app/jobs/csv_upload_job.rb

@@ -1,145 +0,0 @@
-class CsvUploadJob < ActiveJob::Base
-  queue_as Hyrax.config.ingest_queue_name
-  
-  
-	def perform(csv, mvs, collection, admin_set, current_user)
-		@current_user = current_user
-		@csv = CSV.parse(File.read(csv), headers: true, encoding: 'utf-8').map(&:to_hash)
-		@mvs = mvs
-		@collection = Collection.find(collection) rescue nil
-		@admin_set = AdminSet.find(admin_set) rescue nil
-		@works = []
-		@files = {}
-		@csv.each do |row|
-			type = row.first.last
-			if type.nil?
-			  next
-			elsif(type.include? "Work")
-				@works << row
-				@files[@works.length] = []
-				@worktype = type.dup
-			elsif(type.include? "File")
-				row.delete("object_type")
-				@files[@works.length] << row
-			end
-		end
-		create_works
-	end
-	
-	private
-
-	def work_form
-		Module.const_get("Hyrax::#{@worktype}Form") rescue nil || Module.const_get("Hyrax::Forms::WorkForm")
-	end
-
-	def file_form
-		Module.const_get("Hyrax::FileSetForm") rescue nil || Module.const_get("Hyrax::Forms::FileSetEditForm")
-	end
-
-	def secondary_terms form_name
-		form_name.terms - form_name.required_fields -
-				[:visibility_during_embargo, :embargo_release_date,
-				 :visibility_after_embargo, :visibility_during_lease,
-				 :lease_expiration_date, :visibility_after_lease, :visibility,
-				 :thumbnail_id, :representative_id, :ordered_member_ids,
-				 :collection_ids, :in_works_ids, :admin_set_id, :files, :source, :member_of_collection_ids]
-	end
-	
-		def create_file_from_url(url, file_name, work, file_data)
-			::FileSet.new(import_url: url, label: file_name) do |fs|
-			  fs.save
-				actor = Hyrax::Actors::FileSetActor.new(fs, @current_user)
-				actor.create_metadata#(work, visibility: work.visibility)
-				actor.attach_file_to_work(work)
-				#byebug
-				fs.attributes = file_data
-				fs.save!
-				uri = URI.parse(url.gsub(' ','%20'))
-				if uri.scheme == 'file'
-					IngestLocalFileJob.perform_later(fs, uri.path.gsub('%20',' '), @current_user)
-				else
-					ImportUrlJob.perform_later(fs, log(actor.user))
-				end
-			end
-		end
-		#
-		def load_metadata(fs, file_array)
-			file_array.each do |line|
-				fileset = fs
-				index = -1
-				line.each do |data|
-					index = index + 1
-					next if index==0
-					if @csv.headers[index] == "visibility"
-						fileset.visibility = data
-					elsif @csv.headers[index] == "depositor"
-						fileset.depositor = data
-					else
-						data_arr = data.split @mvs
-						fileset[@csv.headers[index]] = data_arr
-					end
-				end
-				fileset.save
-			end
-		end
-
-		def create_works
-			index = 1
-			@works.each do |work_data|
-				work = Object.const_get(work_data.first.last).new#delete("object_type")).new
-				status_after, embargo_date, lease_date = nil, nil, nil
-				final_work_data = create_data work_data, work_form, work
-				work.apply_depositor_metadata(@current_user)
-				work.attributes = final_work_data
-				work.member_of_collections = [@collection] if @collection
-				work.admin_set = @admin_set if @admin_set
-				work.save
-				create_files(work, index)
-				index+=1
-			end
-		end
-	  
-		def create_data data, type, object
-		  final_data = {}
-		  accepted_terms = type.required_fields + secondary_terms(type)
-      data.each do |key, att|
-        if(att.nil? || att.empty? || key.to_s.include?("object_type") || !accepted_terms.include?(key.to_sym) )
-          next
-        elsif(object.send(key).nil?)
-          final_data[key] = att
-        else
-          final_data[key] = att.split @mvs
-        end
-      end
-      final_data
-		end
-		
-		def create_lease visibility, status_after, date
-		  lease = Hydra::AccessControls::Lease.new(visibility_during_lease: visibility,
-						visibility_after_lease: @status_after, lease_expiration_date: @lease_date)
-			lease.save
-		end
-		
-		def create_embargo visibility
-		  embargo = Hydra::AccessControls::Embargo.new
-      embargo.visibility_during_embargo = visibility
-      embargo.visibility_after_embargo = @status_after
-      embargo.embargo_release_date = @embargo_date
-      embargo.save
-		end
-		
-		def create_files(work, index)
-		  file = FileSet.new
-			@files[index].each do |file_data|
-				url = file_data.delete('url')
-				title = file_data.delete('title')
-				final_file_data = create_data file_data, file_form, file
-				create_file_from_url(url, title, work, final_file_data)
-			end
-		end
-
-    def log(user)
-        Hyrax::Operation.create!(user: user,
-                                            operation_type: "Attach Remote File")
-    end 
-end