zizia 4.0.3.alpha.01 → 4.0.4.alpha.01

data/app/uploaders/zizia/csv_manifest_uploader.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'carrierwave'
+
+module Zizia
+  class CsvManifestUploader < CarrierWave::Uploader::Base
+    # Choose what kind of storage to use for this uploader:
+    storage :file
+
+    # Process calls that method whenever a file is uploaded.
+    process :validate_csv
+
+    # The directory where the csv manifest will be stored.
+    def store_dir
+      manifests_path || Rails.root.join('tmp', 'csv_uploads')
+    end
+
+    def cache_dir
+      manifests_cache_path || Rails.root.join('tmp', 'csv_uploads_cache')
+    end
+
+    # Add a white list of extensions which are allowed to be uploaded.
+    # For images you might use something like this:
+    #   %w(jpg jpeg gif png)
+    def extension_whitelist
+      %w[csv]
+    end
+
+    # These are stored in memory only, not persisted
+    def errors
+      @validator ? @validator.errors : []
+    end
+
+    # These are stored in memory only, not persisted
+    def warnings
+      @validator ? @validator.warnings : []
+    end
+
+    def records
+      @validator ? @validator.record_count : 0
+    end
+
+    private
+
+      def manifests_path
+        return false if ENV['CSV_MANIFESTS_PATH'].nil?
+        return false unless File.directory?(ENV['CSV_MANIFESTS_PATH'])
+        ENV['CSV_MANIFESTS_PATH']
+      end
+
+      def manifests_cache_path
+        return false if ENV['CSV_MANIFESTS_CACHE_PATH'].nil?
+        return false unless File.directory?(ENV['CSV_MANIFESTS_CACHE_PATH'])
+        ENV['CSV_MANIFESTS_CACHE_PATH']
+      end
+
+      def validate_csv
+        @validator = CsvManifestValidator.new(self)
+        @validator.validate
+      end
+  end
+end

data/app/uploaders/zizia/csv_manifest_validator.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+# Validate a CSV file.
+#
+# Don't put expensive validations in this class.
+# This is meant to be used for running a few quick
+# validations before starting a CSV-based import.
+# It will be called during the HTTP request/response,
+# so long-running validations will make the page load
+# slowly for the user.  Any validations that are slow
+# should be run in background jobs during the import
+# instead of here.
+module Zizia
+  class CsvManifestValidator
+    # @param manifest_uploader [CsvManifestUploader] The manifest that's mounted to a CsvImport record.  See carrierwave gem documentation.  This is basically a wrapper for the CSV file.
+    def initialize(manifest_uploader)
+      @csv_file = manifest_uploader.file
+      @errors = []
+      @warnings = []
+    end
+
+    # Errors and warnings for the CSV file.
+    attr_reader :errors, :warnings
+    attr_reader :csv_file
+
+    def validate
+      parse_csv
+      return unless @rows
+
+      missing_headers
+      duplicate_headers
+      unrecognized_headers
+      missing_values
+      invalid_license
+      invalid_resource_type
+      invalid_rights_statement
+    end
+
+    # One record per row
+    def record_count
+      return nil unless @rows
+      @rows.size - 1 # Don't include the header row
+    end
+
+    def delimiter
+      @delimiter ||= default_delimiter
+    end
+    attr_writer :delimiter
+
+    private
+
+      def default_delimiter
+        Zizia::HyraxBasicMetadataMapper.new.delimiter
+      end
+
+      def valid_headers
+        ['title', 'files', 'representative media',
+         'thumbnail', 'rendering', 'depositor',
+         'date_uploaded', 'date_modified', 'label',
+         'relative_path', 'import url', 'resource type',
+         'creator', 'contributor', 'abstract or summary',
+         'keyword', 'license', 'rights statement',
+         'publisher', 'date created', 'subject',
+         'language', 'identifier', 'location',
+         'related url', 'bibliographic_citation',
+         'source', 'visibility']
+      end
+
+      def parse_csv
+        @rows = CSV.read(csv_file.path)
+        @headers = @rows.first || []
+        @transformed_headers = @headers.map { |header| header.downcase.strip }
+      rescue
+        @errors << 'We are unable to read this CSV file.'
+      end
+
+      def missing_headers
+        required_headers.each do |header|
+          next if @transformed_headers.include?(header)
+          @errors << "Missing required column: \"#{header.titleize}\".  Your spreadsheet must have this column."
+        end
+      end
+
+      def required_headers
+        ['title', 'creator', 'keyword', 'rights statement', 'visibility', 'files']
+      end
+
+      def duplicate_headers
+        duplicates = []
+        sorted_headers = @transformed_headers.sort
+        sorted_headers.each_with_index do |x, i|
+          duplicates << x if x == sorted_headers[i + 1]
+        end
+        duplicates.uniq.each do |header|
+          @errors << "Duplicate column names: You can have only one \"#{header.titleize}\" column."
+        end
+      end
+
+      # Warn the user if we find any unexpected headers.
+      def unrecognized_headers
+        extra_headers = @transformed_headers - valid_headers
+        extra_headers.each do |header|
+          @warnings << "The field name \"#{header}\" is not supported.  This field will be ignored, and the metadata for this field will not be imported."
+        end
+      end
+
+      def missing_values
+        column_numbers = required_headers.map { |header| @transformed_headers.find_index(header) }.compact
+
+        @rows.each_with_index do |row, i|
+          column_numbers.each_with_index do |column_number, j|
+            next unless row[column_number].blank?
+            @errors << "Missing required metadata in row #{i + 1}: \"#{required_headers[j].titleize}\" field cannot be blank"
+          end
+        end
+      end
+
+      # Only allow valid license values expected by Hyrax.
+      # Otherwise the app throws an error when it displays the work.
+      def invalid_license
+        validate_values('license', :valid_licenses)
+      end
+
+      def invalid_resource_type
+        validate_values('resource type', :valid_resource_types)
+      end
+
+      def invalid_rights_statement
+        validate_values('rights statement', :valid_rights_statements)
+      end
+
+      def valid_licenses
+        @valid_license_ids ||= Hyrax::LicenseService.new.authority.all.select { |license| license[:active] }.map { |license| license[:id] }
+      end
+
+      def valid_resource_types
+        @valid_resource_type_ids ||= Qa::Authorities::Local.subauthority_for('resource_types').all.select { |term| term[:active] }.map { |term| term[:id] }
+      end
+
+      def valid_rights_statements
+        @valid_rights_statement_ids ||= Qa::Authorities::Local.subauthority_for('rights_statements').all.select { |term| term[:active] }.map { |term| term[:id] }
+      end
+
+      # Make sure this column contains only valid values
+      def validate_values(header_name, valid_values_method)
+        column_number = @transformed_headers.find_index(header_name)
+        return unless column_number
+
+        @rows.each_with_index do |row, i|
+          next if i.zero? # Skip the header row
+          next unless row[column_number]
+
+          values = row[column_number].split(delimiter)
+          valid_values = method(valid_values_method).call
+          invalid_values = values.select { |value| !valid_values.include?(value) }
+
+          invalid_values.each do |value|
+            @errors << "Invalid #{header_name.titleize} in row #{i + 1}: #{value}"
+          end
+        end
+      end
+  end
+end

data/app/views/layouts/zizia/application.html.erb

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>Zizia</title>
+  <%= stylesheet_link_tag    "zizia/application", media: "all" %>
+  <%= javascript_include_tag "zizia/application" %>
+  <%= csrf_meta_tags %>
+</head>
+<body>
+
+<%= yield %>
+
+</body>
+</html>

data/app/views/zizia/csv_imports/_actions.html.erb

@@ -0,0 +1,3 @@
+<div class="actions">
+  <%= form.submit 'Preview Import',  class: "btn btn-primary btn-lg" %>
+</div>

data/app/views/zizia/csv_imports/_collection_selection.html.erb

@@ -0,0 +1,3 @@
+<div class="form-group">
+  <%= form.select :fedora_collection_id, options_for_select(collections_for_select), {}, class: "form-control", id: "fedora_collection_id"  %>
+</div>

data/app/views/zizia/csv_imports/_error.html.erb

@@ -0,0 +1,10 @@
+<% if csv_import.errors.any? %>
+  <div id="error_explanation">
+    <h2><%= pluralize(csv_import.errors.size, "error") %> prohibited this csv_import from being saved:</h2>
+    <ul>
+      <% csv_import.errors.full_messages.each do |message| %>
+        <li><%= message %></li>
+      <% end %>
+    </ul>
+  </div>
+<% end %>

data/app/views/zizia/csv_imports/_file_upload.html.erb

@@ -0,0 +1,3 @@
+<%= form.file_field :manifest, required: true, id: 'file-upload', class: "file-upload form-control", onchange: "Zizia.displayUploadedFile()" %>
+<label for="file-upload" class="btn btn-lg btn-primary"><span class="glyphicon glyphicon-upload"></span> Upload Your CSV</label>
+<div id="file-upload-display"></div>

data/app/views/zizia/csv_imports/_form.html.erb

@@ -0,0 +1,83 @@
+<%= form_with(model: csv_import, local: true, html: { multipart: true }, url: preview_csv_import_path) do |form| %>
+  <div class="panel panel-default">
+    <div class="panel-body">
+      <div class="row">
+        <div class="col-md-8">
+          <h2> Getting Started </h2>
+        </div>
+      </div>
+      <div class="row">
+        <div class="col-md-4">
+          <div class="well well-lg">
+            <p><b>Read the CSV Field Guide</b> to learn what is required in your CSV.</p>
+            <div class="text-center">
+              <a href="/importer_documentation/guide" class="btn btn-default btn-lg"><i class="glyphicon glyphicon-book"></i> CSV Field Guide</a>
+            </div>
+          </div>
+        </div>
+        <div class="col-md-4">
+          <div class="well well-lg">
+            <p><b>Download a CSV template</b> to get started.</b></p>
+            <div class="text-center">
+              <a href="/importer_documentation/csv" data-turbolinks="false" class="btn btn-default btn-lg"><i class="glyphicon glyphicon-list"></i> Download a CSV Template</a>
+            </div>
+          </div>
+        </div>
+      </div>
+      <hr/>
+
+      <% if collections? %>
+        <div class="row">
+          <div class="col-md-8">
+            <h2> Upload Your CSV </h2>
+          </div>
+        </div>
+        <div class="row">
+          <div class="col-md-4">
+            <div class="well well-lg">
+              <p> <b>Upload a CSV</b> that has your content. </p>
+              <div class="text-center">
+                <%= render "file_upload", form: form  %>
+              </div>
+            </div>
+          </div>
+          <div class="col-md-4">
+            <div class="well well-lg">
+              <p> <b>Choose a collection</b>. All imports need to be associated with a collection. </p>
+              <div class="text-center">
+                <%= render "collection_selection", form: form %>
+              </div>
+            </div>
+          </div>
+          <div class="col-md-4">
+            <div class="well well-lg">
+              <p> <b>Import Behavior</b>. Choose how you would like existing IDs to be handled.</p>
+              <div class="text-center">
+                <%= render "update_actor_stack", form: form %>
+              </div>
+            </div>
+          </div>
+          <div class="col-md-4">
+            <div class="well well-lg">
+              <p> <b>Preview your import</b> before starting the process. </p>
+              <div class="text-center">
+                <%= render "actions", form: form %>
+              </div>
+            </div>
+          </div>
+        </div>
+
+        <div class="row">
+          <div class="col-md-4">
+            <%= render "error", csv_import: csv_import %>
+          </div>
+          <div class="col-md-4">
+          </div>
+        </div>
+
+
+        <%= form.hidden_field :manifest_cache %>
+      <% else %>
+        <%= render 'no_collection' %>
+      <% end %>
+<% end %>

data/app/views/zizia/csv_imports/_no_collection.html.erb

@@ -0,0 +1,7 @@
+<div class="row">
+  <div class="col-md-4">
+    <div class="well well-lg no-collection">
+      You must <a href="/dashboard/collections">create a collection</a> before importing content.
+    </div>
+  </div>
+</div>

data/app/views/zizia/csv_imports/_record_count.html.erb

@@ -0,0 +1,8 @@
+<div class="row">
+  <div class="col-md-8">
+    <div class="alert alert-success">
+      <p> This import will create or update <%= @csv_import.manifest_records %> records. </p>
+      <p> If you only want to import new records you will be able to choose that option after continuing. </p>
+    </div>
+  </div>
+</div>

data/app/views/zizia/csv_imports/_start_import_form.html.erb

@@ -0,0 +1,10 @@
+<%= form_with(model: csv_import, local: true, html: { multipart: true }) do |form| %>
+
+  <%= form.hidden_field :manifest_cache %>
+  <%= form.hidden_field :fedora_collection_id, value: csv_import.fedora_collection_id %>
+  <%= form.hidden_field :update_actor_stack, value: csv_import.update_actor_stack %>
+
+  <div class="actions">
+    <%= form.submit 'Start Import', class: 'btn btn-lg btn-primary' %>
+  </div>
+<% end %>

data/app/views/zizia/csv_imports/_update_actor_stack.html.erb

@@ -0,0 +1,5 @@
+<div class="form-group">
+  <%= form.select :update_actor_stack, options_for_select([['Update metadata for any existing IDs', 'HyraxMetadataOnly'],
+                                                           ['Only create works for new IDs', 'HyraxOnlyNew' ],
+                                                           ['Update metadata and files or create new works as required', 'HyraxDelete']]), {}, class: "form-control", id: "update_actor_stack_id"  %>
+</div>

data/app/views/zizia/csv_imports/index.html.erb

@@ -0,0 +1,21 @@
+<p id="notice"><%= notice %></p>
+
+<h2>CSV Imports</h2>
+
+<table>
+  <thead>
+    <tr>
+      <th colspan="3"></th>
+    </tr>
+  </thead>
+
+  <tbody>
+    <% @csv_imports.each do |csv_import| %>
+      <tr>
+        <% link_text = csv_import.manifest.file ? File.basename(csv_import.manifest.to_s) : 'show' %>
+        <td><%= link_to link_text, csv_import %></td>
+      </tr>
+    <% end %>
+  </tbody>
+</table>
+

data/app/views/zizia/csv_imports/new.html.erb

@@ -0,0 +1,2 @@
+<h2><span class="glyphicon glyphicon-cloud-upload"></span> Begin Your CSV Import </h2>
+<%= render 'form', csv_import: @csv_import %>

data/app/views/zizia/csv_imports/preview.html.erb

@@ -0,0 +1,76 @@
+<h2><span class="glyphicon glyphicon-cloud-upload"></span> Preview your CSV Import</h2>
+<div class="panel panel-default">
+  <div class="panel-body">
+    <div id='csv_info'>
+      <% if @csv_import.manifest.file %>
+        <div class="row">
+          <div class="col-md-4">
+            <label> CSV Manifest: </label>
+            <%= File.basename(@csv_import.manifest.to_s) %>
+          </div>
+          <div class="col-md-4">
+            <label> Collection: </label>
+            <%= ActiveFedora::SolrService.query("id:#{@csv_import.fedora_collection_id}")[0]['title_tesim'][0] %>
+          </div>
+        </div>
+      <% end %>
+
+      <%= render 'record_count', locals: { csv_import: @csv_import } %>
+
+      <% unless @csv_import.manifest_errors.empty? %>
+        <div class="row">
+          <div class="col-md-6">
+            <div class="alert alert-danger">
+              <div id='csv_errors'>
+                <b><i class="glyphicon glyphicon-remove-sign"></i> The CSV file has the following errors:</b>
+                <ul>
+                  <% @csv_import.manifest_errors.each do |error| %>
+                    <li> <%= error %> </li>
+                  <% end %>
+                </ul>
+              </div>
+            </div>
+          </div>
+        </div>
+      <% end %>
+
+      <% unless @csv_import.manifest_warnings.empty? %>
+        <div class="row">
+          <div class="col-md-6">
+            <div class="alert alert-warning">
+              <div id='csv_warnings'>
+                <b><i class="glyphicon glyphicon-exclamation-sign"></i> The CSV file has the following warnings:</b>
+                <ul>
+                  <% @csv_import.manifest_warnings.each do |warning| %>
+                    <li> <%= warning %> </li>
+                  <% end %>
+                </ul>
+              </div>
+            </div>
+          </div>
+        </div>
+      <% end %>
+
+      <% if @csv_import.manifest_errors.empty? %>
+        <div class="row">
+          <div class="col-md-2">
+            <%= render 'start_import_form', csv_import: @csv_import, class: "btn btn-large btn-danger" %>
+          </div>
+          <div class="col-md-2">
+            <%= link_to 'Cancel', new_csv_import_path, class: "btn btn-lg btn-danger" %>
+          </div>
+        </div>
+      <% else %>
+        <div class="row">
+          <div class="col-md-4">
+            <div class="well"><p>You will need to correct the errors with the CSV file before you can continue.</p></br>
+              <div class="text-center"><%= link_to 'Try Again', new_csv_import_path, class: 'btn btn-lg btn-primary' %>
+              </div>
+            </div>
+          </div>
+        </div>
+      <% end %>
+    </div>
+  </div>
+</div>
+</div>

data/app/views/zizia/csv_imports/show.html.erb

@@ -0,0 +1,31 @@
+<h2> Importing Records from a CSV File </h2>
+<p id="notice"><%= notice %></p>
+
+<div class="panel panel-default">
+  <div class="panel-body">
+
+    <p>
+    <label> CSV Manifest: </label>
+    <span> <%= File.basename(@csv_import.manifest.to_s) %> </span>
+    <br />
+    <label> Collection: </label>
+    <span>  <%=  ActiveFedora::SolrService.query("id:#{@csv_import.fedora_collection_id}")[0]['title_tesim'][0] %> </span>
+    <br />
+    <label> Uploaded by: </label>
+    <span> <%= @csv_import.user.name %> </span>
+    <br />
+    <label> Start time: </label>
+    <span> <%= @csv_import.created_at %> </span>
+    </p>
+
+    <div class="row">
+      <div class="col-md-6">
+        <div class='well'>
+          <p> Your records will be imported in the background.  To check the current status, please check the background job status page. </p><br />
+          <div class="text-center"> <%= link_to 'Background Job Status', '/sidekiq', class: "btn btn-primary btn-lg" %> </div>
+        </div>
+      </div>
+    </div>
+
+  </div>
+</div>

data/app/views/zizia/importer_documentation/guide.html.erb

@@ -0,0 +1 @@
+<%= render_guide.html_safe %>

data/bin/rails

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+# This command will automatically be run when you run "rails" with Rails gems
+# installed from the root of your application.
+
+ENGINE_ROOT = File.expand_path('../..', __FILE__)
+ENGINE_PATH = File.expand_path('../../lib/zizia/engine', __FILE__)
+APP_PATH = File.expand_path('../../spec/dummy/config/application', __FILE__)
+
+# Set up gems listed in the Gemfile.
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
+require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE'])
+
+require 'rails/all'
+require 'rails/engine/commands'

data/config/routes.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+Zizia::Engine.routes.draw do
+  get 'importer_documentation/guide'
+  get 'importer_documentation/csv'
+
+  post 'csv_imports/preview', as: 'preview_csv_import'
+  get 'csv_imports/preview', to: redirect('csv_imports/new')
+  resources :csv_imports, only: [:index, :show, :new, :create]
+end

data/db/migrate/201901162141281_create_zizia_csv_imports.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+class CreateZiziaCsvImports < ActiveRecord::Migration[5.1]
+  def change
+    create_table :zizia_csv_imports do |t|
+      t.references :user, foreign_key: true
+      t.timestamps
+    end
+  end
+end

data/db/migrate/201901162203121_add_manifest_to_zizia_csv_imports.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+class AddManifestToZiziaCsvImports < ActiveRecord::Migration[5.1]
+  def change
+    add_column :zizia_csv_imports, :manifest, :string
+  end
+end

data/db/migrate/201901241536541_add_fedora_collection_id_to_zizia_csv_imports.rb

@@ -0,0 +1,5 @@
+class AddFedoraCollectionIdToZiziaCsvImports < ActiveRecord::Migration[5.1]
+  def change
+    add_column :zizia_csv_imports, :fedora_collection_id, :string
+  end
+end

data/db/migrate/201901241536542_add_update_actor_stack_to_zizia_csv_imports.rb

@@ -0,0 +1,5 @@
+class AddUpdateActorStackToZiziaCsvImports < ActiveRecord::Migration[5.1]
+  def change
+    add_column :zizia_csv_imports, :update_actor_stack, :string
+  end
+end

data/db/migrate/20190911133657_create_zizia_pre_ingest_files.rb

@@ -0,0 +1,12 @@
+class CreateZiziaPreIngestFiles < ActiveRecord::Migration[5.1]
+  def change
+    create_table :zizia_pre_ingest_files do |t|
+      t.integer :size
+      t.text :row
+      t.integer :row_number
+      t.string :filename
+      t.belongs_to :pre_ingest_work
+      t.timestamps
+    end
+  end
+end

data/db/migrate/20190911134002_create_zizia_pre_ingest_works.rb

@@ -0,0 +1,9 @@
+class CreateZiziaPreIngestWorks < ActiveRecord::Migration[5.1]
+  def change
+    create_table :zizia_pre_ingest_works do |t|
+      t.integer :parent_object
+      t.belongs_to :csv_import_detail
+      t.timestamps
+    end
+  end
+end

data/db/migrate/20190911142616_create_zizia_csv_import_details.rb

@@ -0,0 +1,8 @@
+class CreateZiziaCsvImportDetails < ActiveRecord::Migration[5.1]
+  def change
+    create_table :zizia_csv_import_details do |t|
+      t.belongs_to :csv_import
+      t.timestamps
+    end
+  end
+end

data/db/migrate/20190918214137_drop_users.rb

@@ -0,0 +1,11 @@
+class DropUsers < ActiveRecord::Migration[5.1]
+  def change
+    def up
+      drop_table :users
+    end
+
+    def down
+      fail ActiveRecord::IrreversibleMigration
+    end
+  end
+end

data/db/migrate/20190919145012_user_roles.rb

@@ -0,0 +1,18 @@
+class UserRoles < ActiveRecord::Migration[5.0]
+  def up
+    create_table :roles do |t|
+      t.string :name
+    end
+    create_table :roles_users, :id => false do |t|
+      t.references :role
+      t.references :user
+    end
+    add_index :roles_users, [:role_id, :user_id]
+    add_index :roles_users, [:user_id, :role_id]
+  end
+
+  def down
+    drop_table :roles_users
+    drop_table :roles
+  end
+end

data/db/migrate/20190921153802_add_depositor_id_to_csv_import_detail.rb

@@ -0,0 +1,5 @@
+class AddDepositorIdToCsvImportDetail < ActiveRecord::Migration[5.1]
+  def change
+    add_column :zizia_csv_import_details, :depositor_id, :string
+  end
+end