zizia 4.0.3.alpha.01 → 4.0.4.alpha.01

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1a3b63826827c01a64976c80bff369266b9006772d6bf9c6de4cdcfc8b5ee2d6
-  data.tar.gz: a1fb2e5d4e00db022d50e9637d9369e999a3b4e39c7c03d5f1fc4f294213c715
+  metadata.gz: bece199128061310a649b7d80477b923a8502a9f9efe7df6f17e9d37f6e9949b
+  data.tar.gz: db6340fe00181947086763792e93cf3d8fca56a5991328b019628e3bf24ea794
 SHA512:
-  metadata.gz: 9dc28d1191c9b15d12432facf356ae659b842fb912c16b5bc9d3dd3f929d353c916b22b6a53269953ec75702ef90e8b36674062fcb4ec382d87617630c787a87
-  data.tar.gz: d125f72d2bb4fd2b6154cd5e5ffa0c76da6afdb1c8e8786d6d61d853ba9890246edb0db3dd5f41e6b743dbb2610c1a37fe9dc935f0cf6e9c6400e7ca95eebd4e
+  metadata.gz: c06e3fbeffd55dfb00ffcf6e3178bbedfc51cfca7f0537e9b00cb8e50e14f73b3bab386636b0dba1c245098f34509fbdaa0c2151b7001c963b65bf369610bb1c
+  data.tar.gz: 3e07a46b1045f3d686ec889431194198e7ce5efe85f078b0b787e81ea1d54726aba49b4334eb85aab973e667eafdbbe9d54e6dc3617fafa4e6680701abdf7550

data/.circleci/config.yml

@@ -0,0 +1,45 @@
+version: 2.1
+orbs:
+    samvera: samvera/circleci-orb@0.3.1
+jobs:
+    build:
+        parameters:
+            ruby_version:
+                type: string
+                default: 2.5.3
+            bundler_version:
+                type: string
+                default: 2.0.1
+        executor:
+            name: samvera/ruby_fcrepo_solr_redis
+            ruby_version: << parameters.ruby_version >>
+        working_directory: ~/project
+        parallelism: 1
+        environment:
+          COVERALLS_PARALLEL: true
+        steps:
+            - checkout
+
+            - run:
+                name: install sqlite3
+                command: sudo apt-get update && sudo apt-get install -y libsqlite3-dev
+
+            - samvera/bundle:
+                ruby_version: << parameters.ruby_version >>
+                bundler_version: << parameters.bundler_version >>
+
+            - samvera/rubocop
+
+            - samvera/install_solr_core
+
+            - run:
+                name: install sqlite3
+                command: bundle exec rake db:migrate
+
+            - samvera/parallel_rspec
+workflows:
+    version: 2
+    ci:
+        jobs:
+            - build:
+                name: ruby2-5-3

data/.gitignore

@@ -0,0 +1,17 @@
+pkg
+tmp
+Gemfile.lock
+.byebug_history
+.fcrepo_wrapper_test.yml
+.solr_wrapper_test.yml
+zizia_import.log
+*~
+\#*\#
+/.emacs.desktop
+/.emacs.desktop.lock
+*.elc
+auto-save-list
+tramp
+.\#*
+coverage/
+*.gem

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,80 @@
+inherit_from: .rubocop_todo.yml
+
+inherit_gem:
+  bixby: bixby_default.yml
+
+AllCops:
+  TargetRubyVersion: 2.3
+  Exclude:
+    - 'db/**/*'
+    - 'config/**/*'
+    - 'script/**/*'
+    - 'vendor/**/*'
+    - 'bin/{rails,rake}'
+    - 'spec/dummy/**/*'
+
+Lint/HandleExceptions:
+  Exclude:
+    - 'spec/**/*'
+    - 'lib/zizia/spec/**/*'
+
+Metrics/AbcSize:
+  Exclude:
+    - 'spec/support/hyrax/basic_metadata.rb'
+    - 'lib/zizia/hyrax/hyrax_record_importer.rb'
+    - lib/zizia/importer.rb
+    - lib/zizia/hyrax/hyrax_metadata_only_updater.rb
+    - 'app/importers/modular_importer.rb'
+
+Metrics/BlockLength:
+  Exclude:
+    - 'spec/**/*'
+    - 'lib/zizia/spec/**/*'
+    - 'zizia.gemspec'
+
+Metrics/ClassLength:
+  Enabled: false
+
+Metrics/CyclomaticComplexity:
+  Exclude:
+    - lib/zizia/hyrax/hyrax_basic_metadata_mapper.rb
+    - lib/zizia/hyrax/hyrax_record_importer.rb
+
+Metrics/LineLength:
+  Enabled: false
+
+Metrics/MethodLength:
+  Exclude:
+    - 'spec/support/hyrax/basic_metadata.rb'
+    - 'lib/zizia/hyrax/hyrax_basic_metadata_mapper.rb'
+    - lib/zizia/hyrax/hyrax_record_importer.rb
+    - 'app/importers/modular_importer.rb'
+
+Metrics/PerceivedComplexity:
+  Exclude:
+    - lib/zizia/hyrax_record_importer.rb
+
+RSpec/DescribeClass:
+  Exclude:
+    - 'spec/integration/**/*'
+    - 'spec/*_spec.rb'
+
+RSpec/ExampleLength:
+  Exclude:
+    - 'spec/zizia/hyrax/hyrax_basic_metadata_mapper_spec.rb'
+    - 'spec/integration/import_hyrax_csv.rb'
+    - 'spec/integration/csv_import_detail_spec.rb'
+
+RSpec/MessageSpies:
+  Exclude:
+    - spec/integration/import_csv_spec.rb
+
+RSpec/MultipleExpectations:
+  Exclude:
+    - 'spec/zizia/hyrax/hyrax_basic_metadata_mapper_spec.rb'
+    - 'spec/integration/import_hyrax_csv.rb'
+    - 'spec/controllers/importer_documentation_controller_spec.rb'
+    - 'spec/integration/csv_import_detail_spec.rb'
+
+Style/StructInheritance:
+  Enabled: false

data/.rubocop_todo.yml

@@ -0,0 +1,21 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2019-02-21 09:32:03 -0500 using RuboCop version 0.52.1.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 1
+Metrics/CyclomaticComplexity:
+  Max: 7
+
+# Offense count: 1
+# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
+# URISchemes: http, https
+Metrics/LineLength:
+  Max: 228
+
+# Offense count: 1
+Metrics/PerceivedComplexity:
+  Max: 8

data/.solr_wrapper

@@ -0,0 +1,8 @@
+version: 6.6.5
+port: 8985
+instance_dir: tmp/solr-test
+download_dir: tmp/solr_downloads
+collection:
+    persist: false
+    dir: solr/config
+    name: hydra-test

data/.travis.yml

@@ -0,0 +1,12 @@
+language: ruby
+sudo:     false
+cache:    bundler
+dist:     trusty
+
+rvm:
+  - 2.3.8
+  - 2.4.5
+  - 2.5.3
+script:
+  - RAILS_ENV='test' bundle exec rake db:migrate
+  - bundle exec rake

data/Gemfile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+gemspec
+
+unless ENV['CI']
+  gem 'guard'
+  gem 'pry'
+end

data/Rakefile

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'fcrepo_wrapper'
+require 'solr_wrapper'
+require 'active_fedora/rake_support'
+
+require 'rspec/core/rake_task'
+require 'rubocop/rake_task'
+
+Bundler::GemHelper.install_tasks
+
+desc 'Run style checker'
+RuboCop::RakeTask.new(:rubocop) do |task|
+  task.fail_on_error = true
+end
+
+desc 'Run specs'
+RSpec::Core::RakeTask.new(:spec)
+
+desc 'Run specs with Fedora & Solr servers'
+task :spec_with_server do
+  with_test_server { Rake::Task['spec'].invoke }
+end
+
+desc 'Check style and run specs'
+task ci: %w[rubocop spec_with_server]
+
+task default: :ci
+APP_RAKEFILE = File.expand_path("../spec/dummy/Rakefile", __FILE__)
+load 'rails/tasks/engine.rake'
+
+load 'rails/tasks/statistics.rake'

data/app/assets/config/zizia_manifest.js

@@ -0,0 +1,2 @@
+//= link_directory ../javascripts/zizia .js
+//= link_directory ../stylesheets/zizia .css

data/app/assets/javascripts/zizia/DisplayUploadedFile.es6

@@ -0,0 +1,43 @@
+export default class DisplayUploadedFile {
+  constructor() {
+    this.regexp = /[^a-zA-Z0-9\.\-\+_]/g
+  }
+  replaceWhitespace(input) {
+    return input.replace(this.regexp, '_')
+  }
+
+  requiresEscape(input) {
+    return this.regexp.test(input)
+  }
+
+  displayReplaceMessage(input) {
+    if (this.requiresEscape(input)) {
+      return `<b> Note: </b> Your file name contained spaces, which have been replaced by underscores.
+This will have no effect on your import.`
+    } else {
+      return ''
+    }
+  }
+
+  display() {
+    var fileInput = document.querySelector('#file-upload')
+    var files = fileInput.files
+    for (var i = 0; i < files.length; i++) {
+      var file = files[i]
+      document.querySelector('#file-upload-display').innerHTML = `
+<div class="row">
+  <div class="col-md-12">
+    <div class="well style="
+         background-color: #dff0d8;
+         border-color: #d6e9c6;
+         color: #3c763d;">
+      <p>You sucessfully uploaded this CSV: <b> ${this.replaceWhitespace(file.name)} </b>
+      </p>
+      ${this.displayReplaceMessage(file.name)}
+      <p>
+    </div>
+  </div>
+</div>`
+    }
+  }
+}

data/app/assets/javascripts/zizia/application.js

@@ -0,0 +1,13 @@
+// This is a manifest file that'll be compiled into application.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
+// or any plugin's vendor/assets/javascripts directory can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// compiled file. JavaScript code in this file should be added after the last require_* statement.
+//
+// Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details
+// about supported directives.
+//
+//= require_tree .

data/app/assets/javascripts/zizia/zizia.js

@@ -0,0 +1,6 @@
+var Zizia = {
+  displayUploadedFile: function() {
+    var DisplayUploadedFile = require('zizia/DisplayUploadedFile')
+    new DisplayUploadedFile().display()
+  }
+}

data/app/assets/stylesheets/zizia/_file_upload.scss

@@ -0,0 +1,17 @@
+.file-upload {
+	width: 0.1px;
+	height: 0.1px;
+	opacity: 0;
+	overflow: hidden;
+	position: absolute;
+	z-index: -1;
+}
+
+.file-upload + label {
+  margin-bottom: 1em;
+}
+
+.file-upload:focus + label {
+	outline: 1px dotted #000;
+	outline: -webkit-focus-ring-color auto 5px;
+}

data/app/assets/stylesheets/zizia/application.css

@@ -0,0 +1,15 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any other CSS/SCSS
+ * files in this directory. Styles in this file should be added after the last require_* statement.
+ * It is generally better to create a new file per style scope.
+ *
+ *= require zizia/zizia
+ *= require_self
+ */

data/app/assets/stylesheets/zizia/zizia.scss

@@ -0,0 +1 @@
+@import 'file_upload';

data/app/controllers/zizia/application_controller.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+module Zizia
+  class ApplicationController < ActionController::Base
+    protect_from_forgery with: :exception
+  end
+end

data/app/controllers/zizia/csv_imports_controller.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+module Zizia
+  class CsvImportsController < ::ApplicationController
+    load_and_authorize_resource
+    before_action :load_and_authorize_preview, only: [:preview]
+
+    with_themed_layout 'dashboard'
+
+    def index; end
+
+    def show; end
+
+    def new; end
+
+    # Validate the CSV file and display errors or
+    # warnings to the user.
+    def preview; end
+
+    def create
+      @csv_import.user = current_user
+      preserve_cache
+
+      if @csv_import.save
+        @csv_import.queue_start_job
+        redirect_to @csv_import
+      else
+        render :new
+      end
+    end
+
+    private
+
+      def load_and_authorize_preview
+        @csv_import = CsvImport.new(create_params)
+        authorize! :create, @csv_import
+      end
+
+      def create_params
+        params.fetch(:csv_import, {}).permit(:manifest, :fedora_collection_id, :update_actor_stack)
+      end
+
+      # Since we are re-rendering the form (once for
+      # :new and again for :preview), we need to
+      # manually set the cache, otherwise the record
+      # will lose the manifest file between the preview
+      # and the record save.
+      def preserve_cache
+        return unless params['csv_import']
+        @csv_import.manifest_cache = params['csv_import']['manifest_cache']
+        @csv_import.fedora_collection_id = params['csv_import']['fedora_collection_id']
+        @csv_import.update_actor_stack = params['csv_import']['update_actor_stack']
+      end
+  end
+end

data/app/controllers/zizia/importer_documentation_controller.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Zizia
+  class ImporterDocumentationController < ApplicationController
+    def guide; end
+
+    def csv
+      send_data Zizia::CsvTemplate.new.to_s, type: 'text/csv; charset=utf-8', disposition: 'attachment', filename: 'import_manifest.csv'
+    end
+  end
+end

data/app/helpers/importer_documentation_helper.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module ImporterDocumentationHelper
+  def render_guide
+    renderer = Redcarpet::Render::HTML.new(autolink: true, with_toc_data: true)
+    markdown = Redcarpet::Markdown.new(renderer)
+    if File.exist?(Rails.root.join('app', 'assets', 'markdown', 'importer_guide.md'))
+      markdown.render(File.open(Rails.root.join('app', 'assets', 'markdown', 'importer_guide.md')).read)
+    else
+      'There is currently no documentation.'
+    end
+  end
+end

data/app/helpers/zizia/application_helper.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+module Zizia
+  module ApplicationHelper
+    def collections_for_select
+      ActiveFedora::SolrService.query('has_model_ssim:Collection').map do |c|
+        [c['title_tesim'][0], c['id']]
+      end
+    end
+
+    def collections?
+      !ActiveFedora::SolrService.query('has_model_ssim:Collection').empty?
+    end
+  end
+end

data/app/importers/modular_importer.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require 'zizia'
+
+class ModularImporter
+  DEDUPLICATION_FIELD = 'identifier'
+
+  def initialize(csv_import)
+    @csv_import = csv_import
+    @csv_file = csv_import.manifest.to_s
+    @collection_id = csv_import.fedora_collection_id
+    @user_id = csv_import.user_id
+    @user_email = User.find(csv_import.user_id).email
+    @row = 0 # Row number doesn't seem to be recorded in the mappper -- there is an attribute for it. record.mapper.row_number is always nil
+  end
+
+  def import
+    raise "Cannot find expected input file #{@csv_file}" unless File.exist?(@csv_file)
+    file = File.open(@csv_file)
+
+    @csv_import.save
+    csv_import_detail = Zizia::CsvImportDetail.find_or_create_by(csv_import_id: @csv_import.id)
+    csv_import_detail.collection_id = @collection_id
+    csv_import_detail.depositor_id = @user_id
+    csv_import_detail.batch_id = @csv_import.id
+    csv_import_detail.deduplication_field = DEDUPLICATION_FIELD
+    csv_import_detail.update_actor_stack = @csv_import.update_actor_stack
+    csv_import_detail.save!
+
+    attrs = {
+      csv_import_detail: csv_import_detail
+    }
+
+    Rails.logger.info "[zizia] event: start_import, batch_id: #{@csv_import.id}, collection_id: #{@collection_id}, user: #{@user_email}"
+    importer = Zizia::Importer.new(parser: Zizia::CsvParser.new(file: file), record_importer: Zizia::HyraxRecordImporter.new(attributes: attrs))
+
+    importer.records.each do |record|
+      pre_ingest_work = Zizia::PreIngestWork.new(csv_import_detail_id: csv_import_detail.id)
+      @row += 1
+
+      pre_ingest_file = Zizia::PreIngestFile.new(row_number: @row,
+                                                 pre_ingest_work: pre_ingest_work,
+                                                 filename: record.mapper.files.first,
+                                                 size: File.join(ENV['IMPORT_PATH'], record.mapper.files.first).size)
+      pre_ingest_work.save!
+      pre_ingest_file.save!
+    end
+    csv_import_detail.save!
+    importer.import
+    file.close
+  end
+end

data/app/jobs/zizia/application_job.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+module Zizia
+  class ApplicationJob < ActiveJob::Base
+  end
+end

data/app/jobs/zizia/start_csv_import_job.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Zizia
+  class StartCsvImportJob < ApplicationJob
+    queue_as :default
+
+    def perform(csv_import_id)
+      csv_import = CsvImport.find csv_import_id
+      Rails.logger.info "[zizia] Starting import with batch ID: #{csv_import_id}"
+      importer = ModularImporter.new(csv_import)
+      importer.import
+    end
+  end
+end

data/app/models/zizia/application_record.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+module Zizia
+  class ApplicationRecord < ActiveRecord::Base
+    self.abstract_class = true
+  end
+end

data/app/models/zizia/csv_import.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Zizia
+  class CsvImport < ApplicationRecord
+    belongs_to :user
+
+    # This is where the CSV file is stored:
+    mount_uploader :manifest, CsvManifestUploader
+
+    delegate :warnings, to: :manifest, prefix: true
+
+    delegate :errors, to: :manifest, prefix: true
+
+    delegate :records, to: :manifest, prefix: true
+
+    def queue_start_job
+      StartCsvImportJob.perform_later(id)
+      # TODO: We'll probably need to store job_id on this record.
+    end
+  end
+end

data/app/models/zizia/csv_import_detail.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+module Zizia
+  class CsvImportDetail < ApplicationRecord
+    after_initialize :set_defaults, unless: :persisted?
+
+    belongs_to :csv_import
+    has_many :pre_ingest_works
+    has_many :pre_ingest_files, through: :pre_ingest_works
+
+    def total_size
+      return 0 if pre_ingest_files.empty?
+      pre_ingest_files.map(&:size).sum
+    end
+
+    def set_defaults
+      self.success_count = 0
+      self.failure_count = 0
+    end
+  end
+end

data/app/models/zizia/pre_ingest_file.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+module Zizia
+  class PreIngestFile < ApplicationRecord
+    belongs_to :pre_ingest_work
+  end
+end

data/app/models/zizia/pre_ingest_work.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+module Zizia
+  class PreIngestWork < ApplicationRecord
+    has_many :pre_ingest_files
+  end
+end