zizia 1.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 488d9a3f2c71f53ed2134084ed2c9acb446b1eae14f4ce9c0a8959557297a733
+  data.tar.gz: c4bc95f1c51e4f619329ad287cdbe15234501bdf0695d8beb42f0d9c99355339
+SHA512:
+  metadata.gz: 5e53b4a49325cab2f77a6fd714e73b9de8c269fff9960daf211897ecad7cd40c81d75ad92ef68a8d8d7bf6bf822b1044e4969888c887bb7e21d4b8198ab2d4e0
+  data.tar.gz: 010b019094941db98acbe9bfd2a92df88f150a45d9cd1de23337dcf74b0525c4f1430715a9c0a87a01dea9b8bf5bc3dac714fb9be4df7fca64ab1e8161fe84d7

data/.gitignore

@@ -0,0 +1,15 @@
+pkg
+tmp
+Gemfile.lock
+.byebug_history
+.fcrepo_wrapper_test.yml
+.solr_wrapper_test.yml
+zizia_import.log
+*~
+\#*\#
+/.emacs.desktop
+/.emacs.desktop.lock
+*.elc
+auto-save-list
+tramp
+.\#*

data/.rubocop.yml

@@ -0,0 +1,65 @@
+inherit_from: .rubocop_todo.yml
+
+inherit_gem:
+  bixby: bixby_default.yml
+
+AllCops:
+  TargetRubyVersion: 2.3
+
+Lint/HandleExceptions:
+  Exclude:
+    - 'spec/**/*'
+    - 'lib/zizia/spec/**/*'
+
+Metrics/AbcSize:
+  Exclude:
+    - 'spec/support/hyrax/basic_metadata.rb'
+    - 'lib/zizia/hyrax_record_importer.rb'
+
+Metrics/BlockLength:
+  Exclude:
+    - 'spec/**/*'
+    - 'lib/zizia/spec/**/*'
+
+Metrics/ClassLength:
+  Enabled: false
+
+Metrics/CyclomaticComplexity:
+  Exclude:
+    - lib/zizia/hyrax_basic_metadata_mapper.rb
+    - lib/zizia/hyrax_record_importer.rb
+
+Metrics/LineLength:
+  Enabled: false
+
+Metrics/MethodLength:
+  Exclude:
+    - 'spec/support/hyrax/basic_metadata.rb'
+    - 'lib/zizia/hyrax_basic_metadata_mapper.rb'
+    - lib/zizia/hyrax_record_importer.rb
+
+Metrics/PerceivedComplexity:
+  Exclude:
+    - lib/zizia/hyrax_record_importer.rb
+
+Naming/AccessorMethodName:
+  Exclude:
+    - lib/zizia/hyrax_record_importer.rb
+
+RSpec/DescribeClass:
+  Exclude:
+    - 'spec/integration/**/*'
+    - 'spec/*_spec.rb'
+
+RSpec/ExampleLength:
+  Exclude:
+    - 'spec/zizia/hyrax_basic_metadata_mapper_spec.rb'
+    - 'spec/integration/import_hyrax_csv.rb'
+
+RSpec/MultipleExpectations:
+  Exclude:
+    - 'spec/zizia/hyrax_basic_metadata_mapper_spec.rb'
+    - 'spec/integration/import_hyrax_csv.rb'
+
+Style/StructInheritance:
+  Enabled: false

data/.rubocop_todo.yml

@@ -0,0 +1,21 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2019-02-21 09:32:03 -0500 using RuboCop version 0.52.1.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 1
+Metrics/CyclomaticComplexity:
+  Max: 7
+
+# Offense count: 1
+# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
+# URISchemes: http, https
+Metrics/LineLength:
+  Max: 228
+
+# Offense count: 1
+Metrics/PerceivedComplexity:
+  Max: 8

data/.solr_wrapper

@@ -0,0 +1,8 @@
+version: 6.6.5
+port: 8985
+instance_dir: tmp/solr-test
+download_dir: tmp/solr_downloads
+collection:
+    persist: false
+    dir: solr/config
+    name: hydra-test

data/.travis.yml

@@ -0,0 +1,11 @@
+language: ruby
+sudo:     false
+cache:    bundler
+dist:     trusty
+
+rvm:
+  - 2.3.8
+  - 2.4.5
+  - 2.5.3
+script:
+  - bundle exec rake

data/Gemfile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+gemspec
+
+unless ENV['CI']
+  gem 'guard'
+  gem 'pry'
+end

data/README.md

@@ -0,0 +1,77 @@
+# Zizia
+
+<table width="100%">
+<tr><td>
+<img alt="Zizia image" src="https://camo.githubusercontent.com/87eafa4a5b6a84802eab583e532bb33881b8a7ab/68747470733a2f2f7777772e706572766572646f6e6b2e636f6d2f77696c64253230666c6f776572732f506172736e69702f476f6c64656e253230416c6578616e646572732f323030383038253230476f6c64656e253230416c6578616e646572253230285a697a69612532306175726561292532302d2532304e47532532302d253230746865253230426f6f6b2532306f6625323057696c64253230466c6f776572732e6a7067" width="500px">
+</td><td>
+Object import for Hyrax. See the <a href="https://www.rubydoc.info/gems/zizia">API documentation</a> for more
+information. See the <a href="https://curationexperts.github.io/zizia/">Getting Started</a> guide for a gentle introduction.
+<br/><br/>
+
+
+[![Gem Version](https://badge.fury.io/rb/zizia.svg)](https://badge.fury.io/rb/zizia)
+[![Build Status](https://travis-ci.org/curationexperts/zizia.svg?branch=master)](https://travis-ci.org/curationexperts/zizia)
+[![Yard Docs](http://img.shields.io/badge/yard-docs-blue.svg)](http://www.rubydoc.info/gems/zizia)
+
+</td></tr>
+</table>
+
+## Usage
+
+In your project's `Gemfile`, add: `gem 'zizia'`, then run `bundle install`.
+
+To do a basic Hyrax import, first ensure that a [work type is registered](http://www.rubydoc.info/github/samvera/hyrax/Hyrax/Configuration#register_curation_concern-instance_method)
+with your `Hyrax` application. Then write a class like this:
+
+```ruby
+require 'zizia'
+
+class MyImporter
+  def initialize(csv_file)
+    @csv_file = csv_file
+    raise "Cannot find expected input file #{csv_file}" unless File.exist?(csv_file)
+  end
+
+  def import
+    attrs = {
+      collection_id: collection_id,     # pass a collection id to the record importer and all records will be added to that collection
+      depositor_id: depositor_id,       # pass a Hyrax user_key here and that Hyrax user will own all objects created during this import
+      deduplication_field: 'identifier' # pass a field with a persistent identifier (e.g., ARK) and it will check to see if a record with that identifier already
+    }                                   # exists, update its metadata if so, and only if it doesn't find a record with that identifier will it make a new object.
+
+    file = File.open(@csv_file)
+    parser = Zizia::CsvParser.new(file: file)
+    record_importer = Zizia::HyraxRecordImporter.new(attributes: attrs)
+    Zizia::Importer.new(parser: parser, record_importer: record_importer).import
+    file.close # unless a block is passed to File.open, the file must be explicitly closed
+  end
+end
+```
+
+You can find [an example csv file for import to Hyrax](https://github.com/curationexperts/zizia/blob/master/spec/fixtures/hyrax/example.csv) in the fixtures directory. Files for attachment should have the filename in a column
+with a heading of `files`, and the location of the files should be specified via an
+environment variables called `IMPORT_PATH`. If `IMPORT_PATH` is not set, `HyraxRecordImporter` will look in `/opt/data` by default.
+
+## Customizing
+To input any kind of file other than CSV, you need to provide a `Parser` (out of the box, we support simple CSV import with `CsvParser`). We will be writing guides about
+how to support custom mappers (for metadata outside of Hyrax's core and basic metadata fields).
+
+This software is primarily intended for use in a [Hyrax](https://github.com/samvera/hyrax) project.
+However, its dependency on `hyrax` is kept strictly optional so most of its code can be reused to
+good effect elsewhere. Note: As of release 2.0, `HyraxBasicMetadataMapper` will be the default mapper.
+
+## Development
+
+```sh
+git clone https://github.com/curationexperts/zizia
+cd zizia
+
+bundle install
+bundle exec rake ci
+```
+
+### RSpec Support
+
+This gem ships with RSpec shared examples and other support tools intended to ease testing and ensure
+interoperability of client code. These can be included by adding `require 'zizia/spec'` to a
+`spec_helper.rb` or `rails_helper.rb` file in your application's test suite.

data/Rakefile

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'fcrepo_wrapper'
+require 'solr_wrapper'
+require 'active_fedora/rake_support'
+
+require 'rspec/core/rake_task'
+require 'rubocop/rake_task'
+
+Bundler::GemHelper.install_tasks
+
+desc 'Run style checker'
+RuboCop::RakeTask.new(:rubocop) do |task|
+  task.fail_on_error = true
+end
+
+desc 'Run specs'
+RSpec::Core::RakeTask.new(:spec)
+
+desc 'Run specs with Fedora & Solr servers'
+task :spec_with_server do
+  with_test_server { Rake::Task['spec'].invoke }
+end
+
+desc 'Check style and run specs'
+task ci: %w[rubocop spec_with_server]
+
+task default: :ci

data/docs/_config.yml

@@ -0,0 +1 @@
+theme: jekyll-theme-minimal

data/docs/index.md

@@ -0,0 +1,98 @@
+# Importing to Hyrax with Zizia
+
+## Getting Started
+The simplest use case is importing content that matches [Hyrax's core and basic metadata fields](http://samvera.github.io/metadata_application_profile.html), plus a few extra fields that let Hyrax know how to display the content properly. At a very high level we're going to:
+1. Write a simple import test first
+1. Get the test passing
+1. Write a rake task to run your import with real data
+
+**Note:** This guide assumes that you already have a working Hyrax instance with at least one work type. If you need help doing that, see [the Hyrax documentation](https://github.com/samvera/hyrax#creating-a-hyrax-based-app). This guide is going to assume we're using an `Image` work type and attaching image files, but should be applicable to any work type or kind of attachment.
+
+### 1. Write a simple import test
+1. Make a directory for your importer: `mkdir app/importers`
+1. Make a directory for your importer tests: `mkdir spec/importers`
+1. Make a directory for your fixture files: `mkdir spec/fixtures/images`
+1. Put three small images in `spec/fixtures/images` In this guide, we're using copyright free images from https://www.pexels.com/, `birds.jpg`, `cat.jpg`, and `dog.jpg`.
+1. Make a directory for your CSV fixture files: `mkdir spec/fixtures/csv_import`
+1. Put a file like this in `spec/fixtures/csv_import`:
+  ```
+  title,source,visibility
+  "A Cute Dog",https://www.pexels.com/photo/animal-blur-canine-close-up-551628/,open
+  "An Interesting Cat",https://www.pexels.com/photo/person-holding-white-cat-1383397/,open
+  "A Flock of Birds",https://www.pexels.com/photo/animal-avian-beak-birds-203088/,open
+  ```
+1. Make a file called `spec/importers/modular_importer_spec.rb` that contains this:
+  ```ruby
+    # frozen_string_literal: true
+
+    require 'rails_helper'
+    require 'active_fedora/cleaner'
+
+    RSpec.describe ModularImporter do
+      let(:modular_csv)     { 'spec/fixtures/csv_import/modular_input.csv' }
+      let(:user) { ::User.batch_user }
+
+      before do
+        DatabaseCleaner.clean
+        ActiveFedora::Cleaner.clean!
+      end
+
+      it "imports a csv" do
+        expect { ModularImporter.new(modular_csv).import }.to change { Work.count }.by 3
+      end
+    end
+  ```
+1. Make a file called `app/importers/modular_importer.rb` that contains just enough of an importer class that your test can run and give a meaningful error:
+  ```ruby
+    class ModularImporter
+      def initialize(csv_file)
+        @csv_file = csv_file
+        raise "Cannot find expected input file #{csv_file}" unless File.exist?(csv_file)
+      end
+
+      def import
+      end
+    end
+  ```
+1. Run your test:
+  ```
+  bundle exec rspec spec/importers/modular_importer_spec.rb
+  ```
+  It should fail with a message like
+  ```
+  expected `Work.count` to have changed by 3, but was changed by 0
+  ```
+
+So, at this point, your test is running, but the importer isn't yet creating any records.
+
+### 2. Get the test passing
+1. Add the zizia gem to your `Gemfile` and run bundle update:
+  ```
+    gem `zizia`
+  ```
+2. Edit `app/importer/modular_importer.rb` so it looks like this:
+  ```ruby
+    require 'zizia'
+
+    class ModularImporter
+      def initialize(csv_file)
+        @csv_file = csv_file
+        raise "Cannot find expected input file #{csv_file}" unless File.exist?(csv_file)
+      end
+
+      def import
+        file = File.open(@csv_file)
+        Zizia::Importer.new(parser: Zizia::CsvParser.new(file: file), record_importer: Zizia::HyraxRecordImporter.new).import
+        file.close # Note that we must close any files we open.
+      end
+    end
+  ```
+3. Now your test should pass with output something like this:
+
+  ```
+  ModularImporter
+  Creating record: ["A Cute Dog"].Record created at: jw827b648Record created at: jw827b648Creating record: ["An Interesting Cat"].Record created at: 3n203z084Record created at: 3n203z084Creating record: ["A Flock of Birds"].Record created at: wm117n96bRecord created at: wm117n96b  imports a csv
+
+  Finished in 7.56 seconds (files took 9.06 seconds to load)
+  1 example, 0 failures
+  ```

data/lib/zizia/always_invalid_validator.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Zizia
+  ##
+  # A Validator that always gives an error named `:everytime`.
+  #
+  # @example
+  #   validator = AlwaysInvalidValidator.new
+  #   validator.validate(:anything, :at, :all) # => [Error<#...>]
+  class AlwaysInvalidValidator < Validator
+    ##
+    # @return [Array<Validator::Error>]
+    def validate(*)
+      [Error.new(self, :everytime)]
+    end
+  end
+end

data/lib/zizia/hash_mapper.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Zizia
+  ##
+  # A generic metadata mapper for input records
+  #
+  # Maps from hash accessor syntax (`['title']`) to method call dot syntax (`.title`)
+  #
+  # The fields provided by this mapper are dynamically determined by the fields
+  # available in the provided metadata hash.
+  #
+  # All field values are given as multi-valued arrays.
+  #
+  # @example
+  #   mapper = HashMapper.new
+  #   mapper.fields # => []
+  #
+  #   mapper.metadata = { title: 'Comet in Moominland', author: 'Tove Jansson' }
+  #   mapper.fields # => [:title, :author]
+  #   mapper.title  # => ['Comet in Moominland']
+  #   mapper.author # => ['Tove Jansson']
+  #
+  class HashMapper < MetadataMapper
+    ##
+    # @param meta [#to_h]
+    # @return [Hash<String, String>]
+    def metadata=(meta)
+      @metadata = meta.to_h
+    end
+
+    ##
+    # @return [Enumerable<Symbol>] The fields the mapper can process
+    def fields
+      return [] if metadata.nil?
+      metadata.keys.map(&:to_sym)
+    end
+
+    ##
+    # @see MetadataMapper#map_field
+    def map_field(name)
+      Array(metadata[name.to_s])
+    end
+  end
+end

data/lib/zizia/hyrax_basic_metadata_mapper.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+require 'uri'
+
+module Zizia
+  ##
+  # A mapper for Hyrax metadata.
+  #
+  # Maps from hash accessor syntax (`['title']`) to method call dot syntax (`.title`).
+  #
+  # The fields provided by this mapper are the same as the properties defined in `Hyrax::CoreMetadata` and `Hyrax::BasicMetadata`.
+  #
+  # @note This mapper allows you to set values for all the Hyrax fields, but depending on how you create the records, some of the values might get clobbered.  For example, if you use Hyrax's actor stack to create records, it might overwrite fields like `date_modified` or `depositor`.
+  #
+  # @see HashMapper Parent class for more info and examples.
+  class HyraxBasicMetadataMapper < HashMapper
+    # If your CSV headers don't exactly match the
+    # the method name for the property's setter
+    # method, add a mapping here.
+    # Example: the method name is work.resource_type,
+    # but in the CSV file, the header is
+    # 'resource type' (without the underscore).
+    CSV_HEADERS = {
+      resource_type: 'resource type',
+      description: 'abstract or summary',
+      rights_statement: 'rights statement',
+      date_created: 'date created',
+      based_near: 'location',
+      related_url: 'related url'
+    }.freeze
+
+    ##
+    # @return [Enumerable<Symbol>] The fields the mapper can process.
+    def fields
+      core_fields + basic_fields + [:visibility]
+    end
+
+    # Properties defined with `multiple: false` in
+    # Hyrax should return a single value instead of
+    # an Array of values.
+    def depositor
+      single_value('depositor')
+    end
+
+    def date_modified
+      single_value('date_modified')
+    end
+
+    def label
+      single_value('label')
+    end
+
+    def relative_path
+      single_value('relative_path')
+    end
+
+    def import_url
+      single_value('import_url')
+    end
+
+    # We should accept visibility values that match the UI and transform them into
+    # the controlled vocabulary term expected by Hyrax
+    def visibility
+      case metadata[matching_header('visibility')]&.downcase&.gsub(/\s+/, "")
+      when 'public'
+        'open'
+      when 'open'
+        'open'
+      when 'registered'
+        'authenticated'
+      when "authenticated"
+        'authenticated'
+      when ::Hyrax::Institution.name&.downcase&.gsub(/\s+/, "")
+        'authenticated'
+      when ::Hyrax::Institution.name_full&.downcase&.gsub(/\s+/, "")
+        'authenticated'
+      when 'private'
+        'restricted'
+      when 'restricted'
+        'restricted'
+      else
+        'restricted' # This is the default if nothing else matches
+      end
+    end
+
+    def files
+      map_field('files')
+    end
+
+    ##
+    # @return [String] The delimiter that will be used to split a metadata field into separate values.
+    # @example
+    #   mapper = HyraxBasicMetadataMapper.new
+    #   mapper.metadata = { 'language' => 'English|~|French|~|Japanese' }
+    #   mapper.language = ['English', 'French', 'Japanese']
+    #
+    def delimiter
+      @delimiter ||= '|~|'
+    end
+    attr_writer :delimiter
+
+    ##
+    # @see MetadataMapper#map_field
+    def map_field(name)
+      method_name = name.to_s
+      method_name = CSV_HEADERS[name] if CSV_HEADERS.keys.include?(name)
+      key = matching_header(method_name)
+      Array(metadata[key]&.split(delimiter))
+    end
+
+    protected
+
+      # Some fields should have single values instead
+      # of array values.
+      def single_value(field_name)
+        metadata[matching_header(field_name)]
+      end
+
+      # Lenient matching for headers.
+      # If the user has headers like:
+      #   'Title' or 'TITLE' or 'Title  '
+      # it should match the :title field.
+      def matching_header(field_name)
+        metadata.keys.find do |key|
+          next unless key
+          key.downcase.strip == field_name
+        end
+      end
+
+      # Properties defined in Hyrax::CoreMetadata
+      # Note that date_uploaded is NOT set here, even though it is defined in
+      # Hyrax::CoreMetadata. Hyrax expects to set date_uploaded itself, and
+      # sending a metadata value for that field interferes with Hyrax expected
+      # behavior.
+      def core_fields
+        [:depositor, :title, :date_modified]
+      end
+
+      # Properties defined in Hyrax::BasicMetadata
+      def basic_fields
+        [:label, :relative_path, :import_url,
+         :resource_type, :creator, :contributor,
+         :description, :keyword, :license,
+         :rights_statement, :publisher, :date_created,
+         :subject, :language, :identifier,
+         :based_near, :related_url,
+         :bibliographic_citation, :source]
+      end
+  end
+end