zizia 1.0.1 → 2.0.0.alpha.01

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 488d9a3f2c71f53ed2134084ed2c9acb446b1eae14f4ce9c0a8959557297a733
-  data.tar.gz: c4bc95f1c51e4f619329ad287cdbe15234501bdf0695d8beb42f0d9c99355339
+  metadata.gz: 024af582dbc7b0f9e6a9cef853e26b3822c19de334087eb0d209306e5ae7360b
+  data.tar.gz: 967080be47ce3a04039352847e7eb0913dfa0bfd7adf9cae5f7636cd4f7c1ba7
 SHA512:
-  metadata.gz: 5e53b4a49325cab2f77a6fd714e73b9de8c269fff9960daf211897ecad7cd40c81d75ad92ef68a8d8d7bf6bf822b1044e4969888c887bb7e21d4b8198ab2d4e0
-  data.tar.gz: 010b019094941db98acbe9bfd2a92df88f150a45d9cd1de23337dcf74b0525c4f1430715a9c0a87a01dea9b8bf5bc3dac714fb9be4df7fca64ab1e8161fe84d7
+  metadata.gz: 70104aeca87a2ed4b657af0ed0a2f2990dbea38f76d4ebca567395b64c0392c7e02952955fe7509c612312dfc7da05f90ccd113ec121c68bb6c467bae66ca498
+  data.tar.gz: 6b588a28977d885d025673accd8f33e82792ffb03eb2e465a86f458883ac2e455a89d66f8d59fa4d3ad66ff8e7a065b4b109c9cab0a430d2ac461fd4335d1293

data/.rubocop.yml

@@ -5,6 +5,13 @@ inherit_gem:
 
 AllCops:
   TargetRubyVersion: 2.3
+  Exclude:
+    - 'db/**/*'
+    - 'config/**/*'
+    - 'script/**/*'
+    - 'vendor/**/*'
+    - 'bin/{rails,rake}'
+    - 'spec/dummy/**/*'
 
 Lint/HandleExceptions:
   Exclude:
@@ -20,6 +27,7 @@ Metrics/BlockLength:
   Exclude:
     - 'spec/**/*'
     - 'lib/zizia/spec/**/*'
+    - 'zizia.gemspec'
 
 Metrics/ClassLength:
   Enabled: false
@@ -60,6 +68,7 @@ RSpec/MultipleExpectations:
   Exclude:
     - 'spec/zizia/hyrax_basic_metadata_mapper_spec.rb'
     - 'spec/integration/import_hyrax_csv.rb'
+    - 'spec/controllers/importer_documentation_controller_spec.rb'
 
 Style/StructInheritance:
   Enabled: false

data/.travis.yml

@@ -8,4 +8,5 @@ rvm:
   - 2.4.5
   - 2.5.3
 script:
+  - RAILS_ENV='test' bundle exec rake db:migrate
   - bundle exec rake

data/README.md

@@ -56,9 +56,30 @@ environment variables called `IMPORT_PATH`. If `IMPORT_PATH` is not set, `HyraxR
 To input any kind of file other than CSV, you need to provide a `Parser` (out of the box, we support simple CSV import with `CsvParser`). We will be writing guides about
 how to support custom mappers (for metadata outside of Hyrax's core and basic metadata fields).
 
-This software is primarily intended for use in a [Hyrax](https://github.com/samvera/hyrax) project.
-However, its dependency on `hyrax` is kept strictly optional so most of its code can be reused to
-good effect elsewhere. Note: As of release 2.0, `HyraxBasicMetadataMapper` will be the default mapper.
+## Hyrax User Interface
+
+Zizia can be installed as a Rails Engine in a Hyrax application. To use the Zizia UI in Hyrax:
+
+1. Add the engine to `routes.rb`:
+```
+  mount Zizia::Engine => '/'
+```
+
+2. Add the helpers to the `ApplicationController`
+
+```
+helper Zizia::Engine.helpers
+```
+
+3. Add documentation and sample csv files to your project at
+
+`app/assets/csv/import_manifest.csv`
+
+and
+
+`app/assets/markdown/importer_guide.md`
+
+4. Add links to `/csv_imports/new` and `/importer_documentation/csv` in the Hyrax dashboard
 
 ## Development
 

data/Rakefile

@@ -32,3 +32,7 @@ desc 'Check style and run specs'
 task ci: %w[rubocop spec_with_server]
 
 task default: :ci
+APP_RAKEFILE = File.expand_path("../spec/dummy/Rakefile", __FILE__)
+load 'rails/tasks/engine.rake'
+
+load 'rails/tasks/statistics.rake'

data/app/assets/config/zizia_manifest.js

@@ -0,0 +1,2 @@
+//= link_directory ../javascripts/zizia .js
+//= link_directory ../stylesheets/zizia .css

data/app/assets/javascripts/zizia/application.js

@@ -0,0 +1,13 @@
+// This is a manifest file that'll be compiled into application.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
+// or any plugin's vendor/assets/javascripts directory can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// compiled file. JavaScript code in this file should be added after the last require_* statement.
+//
+// Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details
+// about supported directives.
+//
+//= require_tree .

data/app/assets/stylesheets/zizia/application.css

@@ -0,0 +1,15 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any other CSS/SCSS
+ * files in this directory. Styles in this file should be added after the last require_* statement.
+ * It is generally better to create a new file per style scope.
+ *
+ *= require_tree .
+ *= require_self
+ */

data/app/controllers/zizia/application_controller.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+module Zizia
+  class ApplicationController < ActionController::Base
+    protect_from_forgery with: :exception
+  end
+end

data/app/controllers/zizia/csv_imports_controller.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+module Zizia
+  class CsvImportsController < ::ApplicationController
+    load_and_authorize_resource
+    before_action :load_and_authorize_preview, only: [:preview]
+
+    with_themed_layout 'dashboard'
+
+    def index; end
+
+    def show; end
+
+    def new; end
+
+    # Validate the CSV file and display errors or
+    # warnings to the user.
+    def preview; end
+
+    def create
+      @csv_import.user = current_user
+      preserve_cache
+
+      if @csv_import.save
+        @csv_import.queue_start_job
+        redirect_to @csv_import
+      else
+        render :new
+      end
+    end
+
+    private
+
+      def load_and_authorize_preview
+        @csv_import = CsvImport.new(create_params)
+        authorize! :create, @csv_import
+      end
+
+      def create_params
+        params.fetch(:csv_import, {}).permit(:manifest, :fedora_collection_id)
+      end
+
+      # Since we are re-rendering the form (once for
+      # :new and again for :preview), we need to
+      # manually set the cache, otherwise the record
+      # will lose the manifest file between the preview
+      # and the record save.
+      def preserve_cache
+        return unless params['csv_import']
+        @csv_import.manifest_cache = params['csv_import']['manifest_cache']
+        @csv_import.fedora_collection_id = params['csv_import']['fedora_collection_id']
+      end
+  end
+end

data/app/controllers/zizia/importer_documentation_controller.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Zizia
+  class ImporterDocumentationController < ApplicationController
+    def guide; end
+
+    def csv
+      return unless File.exist?(Rails.root.join('app', 'assets', 'csv', 'import_manifest.csv'))
+      send_file Rails.root.join('app', 'assets', 'csv', 'import_manifest.csv'), type: 'text/csv; charset=utf-8', disposition: 'attachment', filename: 'import_manifest.csv'
+    end
+  end
+end

data/app/helpers/importer_documentation_helper.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module ImporterDocumentationHelper
+  def render_guide
+    renderer = Redcarpet::Render::HTML.new(autolink: true, with_toc_data: true)
+    markdown = Redcarpet::Markdown.new(renderer)
+    if File.exist?(Rails.root.join('app', 'assets', 'markdown', 'importer_guide.md'))
+      markdown.render(File.open(Rails.root.join('app', 'assets', 'markdown', 'importer_guide.md')).read)
+    else
+      'There is currently no documentation.'
+    end
+  end
+end

data/app/helpers/zizia/application_helper.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+module Zizia
+  module ApplicationHelper
+    def collections_for_select
+      ActiveFedora::SolrService.query('has_model_ssim:Collection').map do |c|
+        [c['title_tesim'][0], c['id']]
+      end
+    end
+
+    def collections?
+      !ActiveFedora::SolrService.query('has_model_ssim:Collection').empty?
+    end
+  end
+end

data/app/jobs/zizia/application_job.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+module Zizia
+  class ApplicationJob < ActiveJob::Base
+  end
+end

data/app/jobs/zizia/start_csv_import_job.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Zizia
+  class StartCsvImportJob < ApplicationJob
+    queue_as :default
+
+    def perform(csv_import_id)
+      csv_import = CsvImport.find csv_import_id
+      log_stream = Zizia.config.default_info_stream
+      log_stream << "Starting import with batch ID: #{csv_import_id}"
+      importer = ModularImporter.new(csv_import)
+      importer.import
+      endcs
+    end
+  end
+end

data/app/mailers/zizia/application_mailer.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+module Zizia
+  class ApplicationMailer < ActionMailer::Base
+    default from: 'from@example.com'
+    layout 'mailer'
+  end
+end

data/app/models/zizia/application_record.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+module Zizia
+  class ApplicationRecord < ActiveRecord::Base
+    self.abstract_class = true
+  end
+end

data/app/models/zizia/csv_import.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Zizia
+  class CsvImport < ApplicationRecord
+    belongs_to :user
+
+    # This is where the CSV file is stored:
+    mount_uploader :manifest, CsvManifestUploader
+
+    delegate :warnings, to: :manifest, prefix: true
+
+    delegate :errors, to: :manifest, prefix: true
+
+    delegate :records, to: :manifest, prefix: true
+
+    def queue_start_job
+      StartCsvImportJob.perform_later(id)
+      # TODO: We'll probably need to store job_id on this record.
+    end
+  end
+end

data/app/uploaders/zizia/csv_manifest_uploader.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require 'carrierwave'
+
+module Zizia
+  class CsvManifestUploader < CarrierWave::Uploader::Base
+    # Choose what kind of storage to use for this uploader:
+    storage :file
+
+    # Process calls that method whenever a file is uploaded.
+    process :validate_csv
+
+    # The directory where the csv manifest will be stored.
+    def store_dir
+      configured_upload_path + "#{model.class.to_s.underscore}/#{mounted_as}/#{model.id}"
+    end
+
+    def cache_dir
+      configured_cache_path + "#{model.class.to_s.underscore}/#{mounted_as}/#{model.id}"
+    end
+
+    # Add a white list of extensions which are allowed to be uploaded.
+    # For images you might use something like this:
+    #   %w(jpg jpeg gif png)
+    def extension_whitelist
+      %w[csv]
+    end
+
+    # These are stored in memory only, not persisted
+    def errors
+      @validator ? @validator.errors : []
+    end
+
+    # These are stored in memory only, not persisted
+    def warnings
+      @validator ? @validator.warnings : []
+    end
+
+    def records
+      @validator ? @validator.record_count : 0
+    end
+
+    private
+
+      def validate_csv
+        @validator = CsvManifestValidator.new(self)
+        @validator.validate
+      end
+
+      def configured_upload_path
+        ENV['CSV_MANIFESTS_PATH'] || base_path_uploads + 'csv_uploads'
+      end
+
+      def configured_cache_path
+        ENV['CSV_MANIFESTS_CACHE_PATH'] || base_path_cache + 'csv_uploads/cache'
+      end
+
+      def base_path_uploads
+        ENV['TRAVIS'] ? Rails.root : Hyrax.config.upload_path.call
+      end
+
+      def base_path_cache
+        ENV['TRAVIS'] ? Rails.root : Hyrax.config.cache_path.call
+      end
+  end
+end

data/app/views/importer_documentation/guide.html.erb

@@ -0,0 +1 @@
+<%= render_guide.html_safe %>

data/app/views/layouts/zizia/application.html.erb

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>Zizia</title>
+  <%= stylesheet_link_tag    "zizia/application", media: "all" %>
+  <%= javascript_include_tag "zizia/application" %>
+  <%= csrf_meta_tags %>
+</head>
+<body>
+
+<%= yield %>
+
+</body>
+</html>

data/app/views/zizia/csv_imports/_actions.html.erb

@@ -0,0 +1,3 @@
+<div class="actions">
+  <%= form.submit 'Preview Import',  class: "btn btn-primary btn-lg" %>
+</div>

data/app/views/zizia/csv_imports/_collection_selection.html.erb

@@ -0,0 +1,3 @@
+<div class="form-group">
+  <%= form.select :fedora_collection_id, options_for_select(collections_for_select), {}, class: "form-control", id: "fedora_collection_id"  %>
+</div>

data/app/views/zizia/csv_imports/_error.html.erb

@@ -0,0 +1,10 @@
+<% if csv_import.errors.any? %>
+  <div id="error_explanation">
+    <h2><%= pluralize(csv_import.errors.size, "error") %> prohibited this csv_import from being saved:</h2>
+    <ul>
+      <% csv_import.errors.full_messages.each do |message| %>
+        <li><%= message %></li>
+      <% end %>
+    </ul>
+  </div>
+<% end %>

data/app/views/zizia/csv_imports/_file_upload.html.erb

@@ -0,0 +1,3 @@
+<%= form.file_field :manifest, required: true, id: 'file-upload', class: "file-upload form-control", onchange: "Tenejo.displayUploadedFile()" %>
+<label for="file-upload" class="btn btn-lg btn-primary"><span class="glyphicon glyphicon-upload"></span> Upload Your CSV</label>
+<div id="file-upload-display"></div>

data/app/views/zizia/csv_imports/_form.html.erb

@@ -0,0 +1,75 @@
+<%= form_with(model: csv_import, local: true, html: { multipart: true }, url: preview_csv_import_path) do |form| %>
+  <div class="panel panel-default">
+    <div class="panel-body">
+      <div class="row">
+        <div class="col-md-8">
+          <h2> Getting Started </h2>
+        </div>
+      </div>
+      <div class="row">
+        <div class="col-md-4">
+          <div class="well well-lg">
+            <p><b>Read the CSV Field Guide</b> to learn what is required in your CSV.</p>
+            <div class="text-center">
+              <a href="/importer_documentation/guide" class="btn btn-default btn-lg"><i class="glyphicon glyphicon-book"></i> CSV Field Guide</a>
+            </div>
+          </div>
+        </div>
+        <div class="col-md-4">
+          <div class="well well-lg">
+            <p><b>Download a CSV template</b> to get started.</b></p>
+            <div class="text-center">
+              <a href="/importer_documentation/csv" data-turbolinks="false" class="btn btn-default btn-lg"><i class="glyphicon glyphicon-list"></i> Download a CSV Template</a>
+            </div>
+          </div>
+        </div>
+      </div>
+      <hr/>
+
+      <% if collections? %>
+        <div class="row">
+          <div class="col-md-8">
+            <h2> Upload Your CSV </h2>
+          </div>
+        </div>
+        <div class="row">
+          <div class="col-md-4">
+            <div class="well well-lg">
+              <p> <b>Upload a CSV</b> that has your content. </p>
+              <div class="text-center">
+                <%= render "file_upload", form: form  %>
+              </div>
+            </div>
+          </div>
+          <div class="col-md-4">
+            <div class="well well-lg">
+              <p> <b>Choose a collection</b>. All imports need to be associated with a collection. </p>
+              <div class="text-center">
+                <%= render "collection_selection", form: form %>
+              </div>
+            </div>
+          </div>
+          <div class="col-md-4">
+            <div class="well well-lg">
+              <p> <b>Preview your import</b> before starting the process. </p>
+              <div class="text-center">
+                <%= render "actions", form: form %>
+              </div>
+            </div>
+          </div>
+        </div>
+
+        <div class="row">
+          <div class="col-md-4">
+            <%= render "error", csv_import: csv_import %>
+          </div>
+          <div class="col-md-4">
+          </div>
+        </div>
+
+
+        <%= form.hidden_field :manifest_cache %>
+      <% else %>
+        <%= render 'no_collection' %>
+      <% end %>
+<% end %>

data/app/views/zizia/csv_imports/_no_collection.html.erb

@@ -0,0 +1,7 @@
+<div class="row">
+  <div class="col-md-4">
+    <div class="well well-lg no-collection">
+      You must <a href="/dashboard/collections">create a collection</a> before importing content.
+    </div>
+  </div>
+</div>

data/app/views/zizia/csv_imports/_record_count.html.erb

@@ -0,0 +1,7 @@
+<div class="row">
+  <div class="col-md-8">
+    <div class="alert alert-success">
+      <p> This import will add <%= @csv_import.manifest_records %>  new records. </p>
+    </div>
+  </div>
+</div>

data/app/views/zizia/csv_imports/_start_import_form.html.erb

@@ -0,0 +1,9 @@
+<%= form_with(model: csv_import, local: true, html: { multipart: true }) do |form| %>
+
+  <%= form.hidden_field :manifest_cache %>
+  <%= form.hidden_field :fedora_collection_id, value: csv_import.fedora_collection_id %>
+
+  <div class="actions">
+    <%= form.submit 'Start Import', class: 'btn btn-lg btn-primary' %>
+  </div>
+<% end %>

data/app/views/zizia/csv_imports/index.html.erb

@@ -0,0 +1,21 @@
+<p id="notice"><%= notice %></p>
+
+<h2>CSV Imports</h2>
+
+<table>
+  <thead>
+    <tr>
+      <th colspan="3"></th>
+    </tr>
+  </thead>
+
+  <tbody>
+    <% @csv_imports.each do |csv_import| %>
+      <tr>
+        <% link_text = csv_import.manifest.file ? File.basename(csv_import.manifest.to_s) : 'show' %>
+        <td><%= link_to link_text, csv_import %></td>
+      </tr>
+    <% end %>
+  </tbody>
+</table>
+

data/app/views/zizia/csv_imports/new.html.erb

@@ -0,0 +1,2 @@
+<h2><span class="glyphicon glyphicon-cloud-upload"></span> Begin Your CSV Import </h2>
+<%= render 'form', csv_import: @csv_import %>

data/app/views/zizia/csv_imports/preview.html.erb

@@ -0,0 +1,76 @@
+<h2><span class="glyphicon glyphicon-cloud-upload"></span> Preview your CSV Import</h2>
+<div class="panel panel-default">
+  <div class="panel-body">
+    <div id='csv_info'>
+      <% if @csv_import.manifest.file %>
+        <div class="row">
+          <div class="col-md-4">
+            <label> CSV Manifest: </label>
+            <%= File.basename(@csv_import.manifest.to_s) %>
+          </div>
+          <div class="col-md-4">
+            <label> Collection: </label>
+            <%= ActiveFedora::SolrService.query("id:#{@csv_import.fedora_collection_id}")[0]['title_tesim'][0] %>
+          </div>
+        </div>
+      <% end %>
+
+      <%= render 'record_count', locals: { csv_import: @csv_import } %>
+
+      <% unless @csv_import.manifest_errors.empty? %>
+        <div class="row">
+          <div class="col-md-6">
+            <div class="alert alert-danger">
+              <div id='csv_errors'>
+                <b><i class="glyphicon glyphicon-remove-sign"></i> The CSV file has the following errors:</b>
+                <ul>
+                  <% @csv_import.manifest_errors.each do |error| %>
+                    <li> <%= error %> </li>
+                  <% end %>
+                </ul>
+              </div>
+            </div>
+          </div>
+        </div>
+      <% end %>
+
+      <% unless @csv_import.manifest_warnings.empty? %>
+        <div class="row">
+          <div class="col-md-6">
+            <div class="alert alert-warning">
+              <div id='csv_warnings'>
+                <b><i class="glyphicon glyphicon-exclamation-sign"></i> The CSV file has the following warnings:</b>
+                <ul>
+                  <% @csv_import.manifest_warnings.each do |warning| %>
+                    <li> <%= warning %> </li>
+                  <% end %>
+                </ul>
+              </div>
+            </div>
+          </div>
+        </div>
+      <% end %>
+
+      <% if @csv_import.manifest_errors.empty? %>
+        <div class="row">
+          <div class="col-md-2">
+            <%= render 'start_import_form', csv_import: @csv_import, class: "btn btn-large btn-danger" %>
+          </div>
+          <div class="col-md-2">
+            <%= link_to 'Cancel', new_csv_import_path, class: "btn btn-lg btn-danger" %>
+          </div>
+        </div>
+      <% else %>
+        <div class="row">
+          <div class="col-md-4">
+            <div class="well"><p>You will need to correct the errors with the CSV file before you can continue.</p></br>
+              <div class="text-center"><%= link_to 'Try Again', new_csv_import_path, class: 'btn btn-lg btn-primary' %>
+              </div>
+            </div>
+          </div>
+        </div>
+      <% end %>
+    </div>
+  </div>
+</div>
+</div>

data/app/views/zizia/csv_imports/show.html.erb

@@ -0,0 +1,31 @@
+<h2> Importing Records from a CSV File </h2>
+<p id="notice"><%= notice %></p>
+
+<div class="panel panel-default">
+  <div class="panel-body">
+
+    <p>
+    <label> CSV Manifest: </label>
+    <span> <%= File.basename(@csv_import.manifest.to_s) %> </span>
+    <br />
+    <label> Collection: </label>
+    <span>  <%=  ActiveFedora::SolrService.query("id:#{@csv_import.fedora_collection_id}")[0]['title_tesim'][0] %> </span>
+    <br />
+    <label> Uploaded by: </label>
+    <span> <%= @csv_import.user.name %> </span>
+    <br />
+    <label> Start time: </label>
+    <span> <%= @csv_import.created_at %> </span>
+    </p>
+
+    <div class="row">
+      <div class="col-md-6">
+        <div class='well'>
+          <p> Your records will be imported in the background.  To check the current status, please check the background job status page. </p><br />
+          <div class="text-center"> <%= link_to 'Background Job Status', sidekiq_web_path, class: "btn btn-primary btn-lg" %> </div>
+        </div>
+      </div>
+    </div>
+
+  </div>
+</div>