zizia 2.0.0.alpha.01 → 2.1.0.alpha.01

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 024af582dbc7b0f9e6a9cef853e26b3822c19de334087eb0d209306e5ae7360b
-  data.tar.gz: 967080be47ce3a04039352847e7eb0913dfa0bfd7adf9cae5f7636cd4f7c1ba7
+  metadata.gz: 35bae6a98cd9938f07fa8de029d79c9628e912f9e8ca31dddd1ac127ac50217c
+  data.tar.gz: fe5e84efeda35a7bd04c76b9261c42059e9e19241359ee02ac8385349288bb0a
 SHA512:
-  metadata.gz: 70104aeca87a2ed4b657af0ed0a2f2990dbea38f76d4ebca567395b64c0392c7e02952955fe7509c612312dfc7da05f90ccd113ec121c68bb6c467bae66ca498
-  data.tar.gz: 6b588a28977d885d025673accd8f33e82792ffb03eb2e465a86f458883ac2e455a89d66f8d59fa4d3ad66ff8e7a065b4b109c9cab0a430d2ac461fd4335d1293
+  metadata.gz: 03b32b69417893e6f3439566f7a0c44e15bd2e4f8f018e1fe930f799db3f9564f23a79ff57727554b1c2b68e6047d7ec2704007612c29f38ab4c9621852b2b6c
+  data.tar.gz: 6e48071e3cb0e90dbeb598fd3e270acc37593376cd0077964a5165ec9ea6da0b6e75040db52fd0f648e8dd117f9eab8e6ca9dab759bd23a9701e990587cb3137

data/.circleci/config.yml

@@ -0,0 +1,44 @@
+version: 2.1
+orbs:
+    samvera: samvera/circleci-orb@dev:fix-bundle-cache
+jobs:
+    build:
+        parameters:
+            ruby_version:
+                type: string
+                default: 2.5.3
+            bundler_version:
+                type: string
+                default: 2.0.1
+        executor:
+            name: samvera/ruby_fcrepo_solr_redis
+            ruby_version: << parameters.ruby_version >>
+        working_directory: ~/project
+        parallelism: 4
+        steps:
+            - checkout
+
+            - run:
+                name: install sqlite3
+                command: sudo apt-get update && sudo apt-get install -y libsqlite3-dev
+
+            - samvera/bundle:
+                ruby_version: << parameters.ruby_version >>
+                bundler_version: << parameters.bundler_version >>
+
+            - samvera/rubocop
+
+            - samvera/install_solr_core
+
+            - run:
+                name: install sqlite3
+                command: bundle exec rake db:migrate
+
+            - samvera/parallel_rspec
+
+workflows:
+    version: 2
+    ci:
+        jobs:
+            - build:
+                name: ruby2-5-3

data/app/importers/modular_importer.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+require 'zizia'
+
+class ModularImporter
+  DEDUPLICATION_FIELD = 'identifier'
+
+  def initialize(csv_import)
+    @csv_import = csv_import
+    @csv_file = csv_import.manifest.file.file.to_s
+    @collection_id = csv_import.fedora_collection_id
+    @user_id = csv_import.user_id
+    @user_email = User.find(csv_import.user_id).email
+  end
+
+  def import
+    raise "Cannot find expected input file #{@csv_file}" unless File.exist?(@csv_file)
+
+    attrs = {
+      collection_id: @collection_id,
+      depositor_id: @user_id,
+      batch_id: @csv_import.id,
+      deduplication_field: DEDUPLICATION_FIELD
+    }
+
+    file = File.open(@csv_file)
+
+    Zizia.config.default_info_stream << "event: start_import, batch_id: #{@csv_import.id}, collection_id: #{@collection_id}, user: #{@user_email}"
+    Zizia::Importer.new(parser: Zizia::CsvParser.new(file: file), record_importer: Zizia::HyraxRecordImporter.new(attributes: attrs)).import
+    file.close
+  end
+end

data/app/jobs/zizia/start_csv_import_job.rb

@@ -10,7 +10,6 @@ module Zizia
       log_stream << "Starting import with batch ID: #{csv_import_id}"
       importer = ModularImporter.new(csv_import)
       importer.import
-      endcs
     end
   end
 end

data/app/uploaders/zizia/csv_manifest_validator.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+# Validate a CSV file.
+#
+# Don't put expensive validations in this class.
+# This is meant to be used for running a few quick
+# validations before starting a CSV-based import.
+# It will be called during the HTTP request/response,
+# so long-running validations will make the page load
+# slowly for the user.  Any validations that are slow
+# should be run in background jobs during the import
+# instead of here.
+module Zizia
+  class CsvManifestValidator
+    # @param manifest_uploader [CsvManifestUploader] The manifest that's mounted to a CsvImport record.  See carrierwave gem documentation.  This is basically a wrapper for the CSV file.
+    def initialize(manifest_uploader)
+      @csv_file = manifest_uploader.file
+      @errors = []
+      @warnings = []
+    end
+
+    # Errors and warnings for the CSV file.
+    attr_reader :errors, :warnings
+    attr_reader :csv_file
+
+    def validate
+      parse_csv
+      return unless @rows
+
+      missing_headers
+      duplicate_headers
+      unrecognized_headers
+      missing_values
+      invalid_license
+      invalid_resource_type
+      invalid_rights_statement
+    end
+
+    # One record per row
+    def record_count
+      return nil unless @rows
+      @rows.size - 1 # Don't include the header row
+    end
+
+    def delimiter
+      @delimiter ||= default_delimiter
+    end
+    attr_writer :delimiter
+
+    private
+
+      def default_delimiter
+        Zizia::HyraxBasicMetadataMapper.new.delimiter
+      end
+
+      def valid_headers
+        ['title', 'files', 'representative media',
+         'thumbnail', 'rendering', 'depositor',
+         'date_uploaded', 'date_modified', 'label',
+         'relative_path', 'import url', 'resource type',
+         'creator', 'contributor', 'abstract or summary',
+         'keyword', 'license', 'rights statement',
+         'publisher', 'date created', 'subject',
+         'language', 'identifier', 'location',
+         'related url', 'bibliographic_citation',
+         'source', 'visibility']
+      end
+
+      def parse_csv
+        @rows = CSV.read(csv_file.path)
+        @headers = @rows.first || []
+        @transformed_headers = @headers.map { |header| header.downcase.strip }
+      rescue
+        @errors << 'We are unable to read this CSV file.'
+      end
+
+      def missing_headers
+        required_headers.each do |header|
+          next if @transformed_headers.include?(header)
+          @errors << "Missing required column: \"#{header.titleize}\".  Your spreadsheet must have this column."
+        end
+      end
+
+      def required_headers
+        ['title', 'creator', 'keyword', 'rights statement', 'visibility', 'files']
+      end
+
+      def duplicate_headers
+        duplicates = []
+        sorted_headers = @transformed_headers.sort
+        sorted_headers.each_with_index do |x, i|
+          duplicates << x if x == sorted_headers[i + 1]
+        end
+        duplicates.uniq.each do |header|
+          @errors << "Duplicate column names: You can have only one \"#{header.titleize}\" column."
+        end
+      end
+
+      # Warn the user if we find any unexpected headers.
+      def unrecognized_headers
+        extra_headers = @transformed_headers - valid_headers
+        extra_headers.each do |header|
+          @warnings << "The field name \"#{header}\" is not supported.  This field will be ignored, and the metadata for this field will not be imported."
+        end
+      end
+
+      def missing_values
+        column_numbers = required_headers.map { |header| @transformed_headers.find_index(header) }.compact
+
+        @rows.each_with_index do |row, i|
+          column_numbers.each_with_index do |column_number, j|
+            next unless row[column_number].blank?
+            @errors << "Missing required metadata in row #{i + 1}: \"#{required_headers[j].titleize}\" field cannot be blank"
+          end
+        end
+      end
+
+      # Only allow valid license values expected by Hyrax.
+      # Otherwise the app throws an error when it displays the work.
+      def invalid_license
+        validate_values('license', :valid_licenses)
+      end
+
+      def invalid_resource_type
+        validate_values('resource type', :valid_resource_types)
+      end
+
+      def invalid_rights_statement
+        validate_values('rights statement', :valid_rights_statements)
+      end
+
+      def valid_licenses
+        @valid_license_ids ||= Hyrax::LicenseService.new.authority.all.select { |license| license[:active] }.map { |license| license[:id] }
+      end
+
+      def valid_resource_types
+        @valid_resource_type_ids ||= Qa::Authorities::Local.subauthority_for('resource_types').all.select { |term| term[:active] }.map { |term| term[:id] }
+      end
+
+      def valid_rights_statements
+        @valid_rights_statement_ids ||= Qa::Authorities::Local.subauthority_for('rights_statements').all.select { |term| term[:active] }.map { |term| term[:id] }
+      end
+
+      # Make sure this column contains only valid values
+      def validate_values(header_name, valid_values_method)
+        column_number = @transformed_headers.find_index(header_name)
+        return unless column_number
+
+        @rows.each_with_index do |row, i|
+          next if i.zero? # Skip the header row
+          next unless row[column_number]
+
+          values = row[column_number].split(delimiter)
+          valid_values = method(valid_values_method).call
+          invalid_values = values.select { |value| !valid_values.include?(value) }
+
+          invalid_values.each do |value|
+            @errors << "Invalid #{header_name.titleize} in row #{i + 1}: #{value}"
+          end
+        end
+      end
+  end
+end

data/app/views/zizia/csv_imports/show.html.erb

@@ -22,7 +22,7 @@
       <div class="col-md-6">
         <div class='well'>
           <p> Your records will be imported in the background.  To check the current status, please check the background job status page. </p><br />
-          <div class="text-center"> <%= link_to 'Background Job Status', sidekiq_web_path, class: "btn btn-primary btn-lg" %> </div>
+          <div class="text-center"> <%= link_to 'Background Job Status', '/sidekiq', class: "btn btn-primary btn-lg" %> </div>
         </div>
       </div>
     </div>

data/docs/index.md

@@ -14,7 +14,7 @@ The simplest use case is importing content that matches [Hyrax's core and basic
 1. Make a directory for your fixture files: `mkdir spec/fixtures/images`
 1. Put three small images in `spec/fixtures/images` In this guide, we're using copyright free images from https://www.pexels.com/, `birds.jpg`, `cat.jpg`, and `dog.jpg`.
 1. Make a directory for your CSV fixture files: `mkdir spec/fixtures/csv_import`
-1. Put a file like this in `spec/fixtures/csv_import`:
+1. Put a file like this in `spec/fixtures/csv_import/zizia_basic.csv`:
   ```
   title,source,visibility
   "A Cute Dog",https://www.pexels.com/photo/animal-blur-canine-close-up-551628/,open
@@ -26,17 +26,11 @@ The simplest use case is importing content that matches [Hyrax's core and basic
     # frozen_string_literal: true
 
     require 'rails_helper'
-    require 'active_fedora/cleaner'
 
     RSpec.describe ModularImporter do
-      let(:modular_csv)     { 'spec/fixtures/csv_import/modular_input.csv' }
+      let(:modular_csv)     { 'spec/fixtures/csv_import/zizia_basic.csv' }
       let(:user) { ::User.batch_user }
 
-      before do
-        DatabaseCleaner.clean
-        ActiveFedora::Cleaner.clean!
-      end
-
       it "imports a csv" do
         expect { ModularImporter.new(modular_csv).import }.to change { Work.count }.by 3
       end
@@ -96,3 +90,24 @@ So, at this point, your test is running, but the importer isn't yet creating any
   Finished in 7.56 seconds (files took 9.06 seconds to load)
   1 example, 0 failures
   ```
+
+### 3. Import in development and production
+
+A passing test tell us that records are being created, but it's only happening in our test environment right now. Let's write a rake task so we can make it happen in development or production and really see the records we made.
+
+Make a file like this at `lib/tasks/import.rake`:
+
+```ruby
+  namespace :basic_import do
+    desc 'Ingest sample data'
+    task sample: [:environment] do
+      Rake::Task["hyrax:default_admin_set:create"].invoke
+      Rake::Task["hyrax:default_collection_types:create"].invoke
+      Rake::Task["hyrax:workflow:load"].invoke
+      csv_file = Rails.root.join('spec', 'fixtures', 'csv_import', 'zizia_basic.csv')
+      ModularImporter.new(csv_file).import
+    end
+  end
+```
+
+Now you should be able to run `rake basic_import:sample` and see your records appear in Hyrax.

data/lib/zizia/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Zizia
-  VERSION = '2.0.0.alpha.01'
+  VERSION = '2.1.0.alpha.01'
 end

data/solr/config/_rest_managed.json

@@ -0,0 +1,3 @@
+{
+  "initArgs":{},
+  "managedList":[]}

data/solr/config/admin-extra.html

@@ -0,0 +1,31 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- The content of this page will be statically included into the top
+of the admin page.  Uncomment this as an example to see there the content
+will show up.
+
+<hr>
+<i>This line will appear before the first table</i>
+<tr>
+<td colspan="2">
+This row will be appended to the end of the first table
+</td>
+</tr>
+<hr>
+
+-->

data/solr/config/elevate.xml

@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- If this file is found in the config directory, it will only be
+     loaded once at startup.  If it is found in Solr's data
+     directory, it will be re-loaded every commit.
+-->
+
+<elevate>
+ <query text="foo bar">
+  <doc id="1" />
+  <doc id="2" />
+  <doc id="3" />
+ </query>
+ 
+ <query text="ipod">
+   <doc id="MA147LL/A" />  <!-- put the actual ipod at the top -->
+   <doc id="IW-02" exclude="true" /> <!-- exclude this cable -->
+ </query>
+ 
+</elevate>

data/solr/config/mapping-ISOLatin1Accent.txt

@@ -0,0 +1,246 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Syntax:
+#   "source" => "target"
+#     "source".length() > 0 (source cannot be empty.)
+#     "target".length() >= 0 (target can be empty.)
+
+# example:
+#   "??" => "A"
+#   "\u00C0" => "A"
+#   "\u00C0" => "\u0041"
+#   "??" => "ss"
+#   "\t" => " "
+#   "\n" => ""
+
+# ?? => A
+"\u00C0" => "A"
+
+# ?? => A
+"\u00C1" => "A"
+
+# ?? => A
+"\u00C2" => "A"
+
+# ?? => A
+"\u00C3" => "A"
+
+# ?? => A
+"\u00C4" => "A"
+
+# ?? => A
+"\u00C5" => "A"
+
+# ?? => AE
+"\u00C6" => "AE"
+
+# ?? => C
+"\u00C7" => "C"
+
+# ?? => E
+"\u00C8" => "E"
+
+# ?? => E
+"\u00C9" => "E"
+
+# ?? => E
+"\u00CA" => "E"
+
+# ?? => E
+"\u00CB" => "E"
+
+# ?? => I
+"\u00CC" => "I"
+
+# ?? => I
+"\u00CD" => "I"
+
+# ?? => I
+"\u00CE" => "I"
+
+# ?? => I
+"\u00CF" => "I"
+
+# ?? => IJ
+"\u0132" => "IJ"
+
+# ?? => D
+"\u00D0" => "D"
+
+# ?? => N
+"\u00D1" => "N"
+
+# ?? => O
+"\u00D2" => "O"
+
+# ?? => O
+"\u00D3" => "O"
+
+# ?? => O
+"\u00D4" => "O"
+
+# ?? => O
+"\u00D5" => "O"
+
+# ?? => O
+"\u00D6" => "O"
+
+# ?? => O
+"\u00D8" => "O"
+
+# ?? => OE
+"\u0152" => "OE"
+
+# ??
+"\u00DE" => "TH"
+
+# ?? => U
+"\u00D9" => "U"
+
+# ?? => U
+"\u00DA" => "U"
+
+# ?? => U
+"\u00DB" => "U"
+
+# ?? => U
+"\u00DC" => "U"
+
+# ?? => Y
+"\u00DD" => "Y"
+
+# ?? => Y
+"\u0178" => "Y"
+
+# ?? => a
+"\u00E0" => "a"
+
+# ?? => a
+"\u00E1" => "a"
+
+# ?? => a
+"\u00E2" => "a"
+
+# ?? => a
+"\u00E3" => "a"
+
+# ?? => a
+"\u00E4" => "a"
+
+# ?? => a
+"\u00E5" => "a"
+
+# ?? => ae
+"\u00E6" => "ae"
+
+# ?? => c
+"\u00E7" => "c"
+
+# ?? => e
+"\u00E8" => "e"
+
+# ?? => e
+"\u00E9" => "e"
+
+# ?? => e
+"\u00EA" => "e"
+
+# ?? => e
+"\u00EB" => "e"
+
+# ?? => i
+"\u00EC" => "i"
+
+# ?? => i
+"\u00ED" => "i"
+
+# ?? => i
+"\u00EE" => "i"
+
+# ?? => i
+"\u00EF" => "i"
+
+# ?? => ij
+"\u0133" => "ij"
+
+# ?? => d
+"\u00F0" => "d"
+
+# ?? => n
+"\u00F1" => "n"
+
+# ?? => o
+"\u00F2" => "o"
+
+# ?? => o
+"\u00F3" => "o"
+
+# ?? => o
+"\u00F4" => "o"
+
+# ?? => o
+"\u00F5" => "o"
+
+# ?? => o
+"\u00F6" => "o"
+
+# ?? => o
+"\u00F8" => "o"
+
+# ?? => oe
+"\u0153" => "oe"
+
+# ?? => ss
+"\u00DF" => "ss"
+
+# ?? => th
+"\u00FE" => "th"
+
+# ?? => u
+"\u00F9" => "u"
+
+# ?? => u
+"\u00FA" => "u"
+
+# ?? => u
+"\u00FB" => "u"
+
+# ?? => u
+"\u00FC" => "u"
+
+# ?? => y
+"\u00FD" => "y"
+
+# ?? => y
+"\u00FF" => "y"
+
+# ??? => ff
+"\uFB00" => "ff"
+
+# ??? => fi
+"\uFB01" => "fi"
+
+# ??? => fl
+"\uFB02" => "fl"
+
+# ??? => ffi
+"\uFB03" => "ffi"
+
+# ??? => ffl
+"\uFB04" => "ffl"
+
+# ??? => ft
+"\uFB05" => "ft"
+
+# ??? => st
+"\uFB06" => "st"