zizia 1.0.1

data/lib/zizia.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+##
+# Bulk object import for Samvera.
+#
+# == Importers
+#
+# {Importer} is the core class for importing records using {Zizia}.
+# Importers accept a {Parser} and (optionally) a custom {RecordImporter}, and
+# process each record in the given parser (see: {Parser#records}).
+#
+# @example Importing in bulk from a file
+#   parser = Zizia::Parser.for(file: File.new('path/to/file.ext'))
+#
+#   Zizia::Importer.new(parser: parser).import if parser.validate
+#
+# @example A basic configuration
+#   Zizia.config do |config|
+#     # error/info streams must respond to `#<<`
+#     config.default_error_stream = MyErrorStream.new
+#     config.default_info_stream  = STDOUT
+#   end
+#
+module Zizia
+  ##
+  # @yield the current configuration
+  # @yieldparam config [Zizia::Configuration]
+  #
+  # @return [Zizia::Configuration] the current configuration
+  def config
+    yield @configuration if block_given?
+    @configuration
+  end
+  module_function :config
+
+  require 'zizia/log_stream'
+  ##
+  # Module-wide options for `Zizia`.
+  class Configuration
+    ##
+    # @!attribute [rw] default_error_stream
+    #   @return [#<<]
+    # @!attribute [rw] default_info_stream
+    #   @return [#<<]
+    attr_accessor :default_error_stream, :default_info_stream
+
+    def initialize
+      self.default_error_stream = Zizia::LogStream.new
+      self.default_info_stream  = Zizia::LogStream.new
+    end
+  end
+
+  @configuration = Configuration.new
+
+  require 'zizia/version'
+  require 'zizia/metadata_mapper'
+  require 'zizia/hash_mapper'
+  require 'zizia/hyrax_basic_metadata_mapper'
+
+  require 'zizia/importer'
+  require 'zizia/record_importer'
+  require 'zizia/hyrax_record_importer'
+
+  require 'zizia/input_record'
+
+  require 'zizia/validator'
+  require 'zizia/validators/csv_format_validator'
+  require 'zizia/validators/title_validator'
+
+  require 'zizia/parser'
+  require 'zizia/parsers/csv_parser'
+  require 'zizia/metadata_only_stack'
+end

data/spec/fixtures/bad_example.csv

@@ -0,0 +1,2 @@
+title,description
+fake title, "fake description--quote wrap after delimiter then space is invalid"

data/spec/fixtures/example.csv

@@ -0,0 +1,4 @@
+title,description
+Fake Item,A description of a fake item.
+Fake Item with Quoted Description,"Lorem ipsum dolor sit amet, cu mel habeo antiopam, id pri mucius oporteat. No graeco accumsan deterruisset est. Vix te sonet doctus perpetua, mei at odio eius nostrum. Ex postea quidam menandri duo. Rebum ullum cu mel.\nAperiam malorum indoctum ad nec.\nIn duo nonumes accusata, detraxit adipisci philosophia quo id. Ei usu volutpat vituperatoribus. Ut veniam dolorem qui. Ei legendos erroribus usu. Mentitum moderatius ei est, mei eius magna alterum no. Legere vivendum per ad, vim ei putent facilis."
+Fake Item with Unicode Description,"Лорем ипсум долор сит амет, ех мовет толлит модератиус ест. Еу яуидам сенсерит цонсецтетуер про, при иисяуе ерудити цоррумпит ат. Ех усу нусяуам пхаедрум темпорибус, ест ат омнесяуе инструцтиор.\nЯуо ех мелиоре инсоленс праесент, иудицо тантас еурипидис хис ут. Аццусам урбанитас инструцтиор ан еам. Но хас вениам дицунт дебитис, нец ут суас аццусам перицула, нец риденс аетерно виртуте не.\nАт про еним вереар, ут солум юсто меи."

data/spec/fixtures/hyrax/example.csv

@@ -0,0 +1,3 @@
+title,depositor,date_modified,label,relative_path,import_url,resource type,creator,contributor,abstract or summary,keyword,license,rights statement,publisher,date created,subject,language,identifier,location,related url,bibliographic_citation,source
+Work 1 Title,user@example.com,2018-01-01,Work 1 Label,tmp/files,https://example.com,Work 1 Type,Work 1 creator,Work 1 contrib,Desc 1,Key 1,Lic 1,RS 1,Pub 1,2018-06-06,Subj 1,English|~|Japanese,Ident 1,Based 1,https://example.com/related,Bib 1,Source 1
+Work 2 Title,1970-12-21,,Work 2 Label,,,Work 2 Type,,,Desc 2,,,,Pub 2,,Subj 2

data/spec/integration/import_csv_spec.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe 'importing a csv batch', :clean do
+  subject(:importer) { Zizia::Importer.new(parser: parser) }
+  let(:parser)       { Zizia::CsvParser.new(file: file) }
+  let(:file)         { File.open('spec/fixtures/example.csv') }
+
+  load File.expand_path("../../support/shared_contexts/with_work_type.rb", __FILE__)
+  include_context 'with a work type'
+
+  it 'creates a record for each CSV line' do
+    expect { importer.import }.to change { Work.count }.to 3
+  end
+
+  describe 'validation' do
+    context 'with invalid CSV' do
+      let(:file) { File.open('spec/fixtures/bad_example.csv') }
+
+      it 'outputs invalid file notice to error stream' do
+        expect { parser.validate }
+          .to output(/^CSV::MalformedCSVError.*line 2/)
+          .to_stdout_from_any_process
+      end
+    end
+  end
+end

data/spec/integration/import_hyrax_csv.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+require 'spec_helper'
+
+describe 'importing a CSV with Hyrax defaults', :clean do
+  subject(:importer) { Zizia::Importer.new(parser: parser, record_importer: record_importer) }
+  let(:parser) { Zizia::CsvParser.new(file: csv_file) }
+  let(:record_importer) { Zizia::HyraxRecordImporter.new }
+
+  let(:csv_file) { File.open('spec/fixtures/hyrax/example.csv') }
+  after { csv_file.close }
+
+  load File.expand_path("../../support/shared_contexts/with_work_type.rb", __FILE__)
+  include_context 'with a work type'
+
+  it 'creates the record(s)' do
+    expect { importer.import }.to change { Work.count }.to 2
+
+    works = Work.all
+    work1 = works.find { |w| w.title == ['Work 1 Title'] }
+    work2 = works.find { |w| w.title == ['Work 2 Title'] }
+
+    # First Record
+    expect(work1.depositor).to eq 'batchuser@example.com'
+    expect(work1.date_modified).to eq '2018-01-01'
+    expect(work1.label).to eq 'Work 1 Label'
+    expect(work1.relative_path).to eq 'tmp/files'
+    expect(work1.import_url).to eq 'https://example.com'
+    expect(work1.resource_type).to eq ['Work 1 Type']
+    expect(work1.creator).to eq ['Work 1 creator']
+    expect(work1.contributor).to eq ['Work 1 contrib']
+    expect(work1.description).to eq ['Desc 1']
+    expect(work1.keyword).to eq ['Key 1']
+    expect(work1.license).to eq ['Lic 1']
+    expect(work1.rights_statement).to eq ['RS 1']
+    expect(work1.publisher).to eq ['Pub 1']
+    expect(work1.date_created).to eq ['2018-06-06']
+    expect(work1.subject).to eq ['Subj 1']
+
+    # An example with 2 values
+    expect(work1.language).to contain_exactly('English', 'Japanese')
+
+    expect(work1.identifier).to eq ['Ident 1']
+    expect(work1.based_near).to eq ['Based 1']
+    expect(work1.related_url).to eq ['https://example.com/related']
+    expect(work1.bibliographic_citation).to eq ['Bib 1']
+    expect(work1.source).to eq ['Source 1']
+
+    # Second Record
+    expect(work2.depositor).to eq 'batchuser@example.com'
+    expect(work2.date_modified).to be_nil
+    expect(work2.label).to eq 'Work 2 Label'
+    expect(work2.relative_path).to be_nil
+    expect(work2.import_url).to be_nil
+    expect(work2.resource_type).to eq ['Work 2 Type']
+    expect(work2.creator).to eq []
+    expect(work2.contributor).to eq []
+    expect(work2.description).to eq ['Desc 2']
+    expect(work2.keyword).to eq []
+    expect(work2.license).to eq []
+    expect(work2.rights_statement).to eq []
+    expect(work2.publisher).to eq ['Pub 2']
+    expect(work2.date_created).to eq []
+    expect(work2.subject).to eq ['Subj 2']
+    expect(work2.language).to eq []
+    expect(work2.identifier).to eq []
+    expect(work2.based_near).to eq []
+    expect(work2.related_url).to eq []
+    expect(work2.bibliographic_citation).to eq []
+    expect(work2.source).to eq []
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'pry' unless ENV['CI']
+ENV['environment'] ||= 'test'
+
+require 'bundler/setup'
+require 'active_fedora'
+require 'active_fedora/cleaner'
+require 'zizia'
+require 'zizia/spec'
+require 'byebug'
+
+RSpec.configure do |config|
+  config.filter_run focus: true
+  config.run_all_when_everything_filtered = true
+
+  config.before(:each, clean: true) { ActiveFedora::Cleaner.clean! }
+end

data/spec/stdout_stream_spec.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe 'STDOUT as a MessageStream' do
+  subject(:stream) { STDOUT }
+
+  it_behaves_like 'a Zizia::MessageStream'
+end

data/spec/support/hyrax/basic_metadata.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Hyrax
+  module BasicMetadata
+    def self.included(work)
+      work.property :label, predicate: ActiveFedora::RDF::Fcrepo::Model.downloadFilename, multiple: false
+      work.property :relative_path, predicate: ::RDF::URI.new('http://scholarsphere.psu.edu/ns#relativePath'), multiple: false
+      work.property :import_url, predicate: ::RDF::URI.new('http://scholarsphere.psu.edu/ns#importUrl'), multiple: false
+      work.property :resource_type, predicate: ::RDF::Vocab::DC.type
+      work.property :creator, predicate: ::RDF::Vocab::DC11.creator
+      work.property :contributor, predicate: ::RDF::Vocab::DC11.contributor
+      work.property :description, predicate: ::RDF::Vocab::DC11.description
+      work.property :keyword, predicate: ::RDF::Vocab::DC11.relation
+      work.property :license, predicate: ::RDF::Vocab::DC.rights
+      work.property :rights_statement, predicate: ::RDF::Vocab::EDM.rights
+      work.property :publisher, predicate: ::RDF::Vocab::DC11.publisher
+      work.property :date_created, predicate: ::RDF::Vocab::DC.created
+      work.property :subject, predicate: ::RDF::Vocab::DC11.subject
+      work.property :language, predicate: ::RDF::Vocab::DC11.language
+      work.property :identifier, predicate: ::RDF::Vocab::DC.identifier
+
+      # Note: based_near is defined differently here than in Hyrax.
+      work.property :based_near, predicate: ::RDF::Vocab::FOAF.based_near
+
+      work.property :related_url, predicate: ::RDF::RDFS.seeAlso
+      work.property :bibliographic_citation, predicate: ::RDF::Vocab::DC.bibliographicCitation
+      work.property :source, predicate: ::RDF::Vocab::DC.source
+    end
+  end
+end

data/spec/support/hyrax/core_metadata.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Hyrax
+  module CoreMetadata
+    def self.included(work)
+      work.property :depositor, predicate: ::RDF::URI.new('http://id.loc.gov/vocabulary/relators/dpt'), multiple: false
+
+      work.property :title, predicate: ::RDF::Vocab::DC.title
+
+      work.property :date_uploaded, predicate: ::RDF::Vocab::DC.dateSubmitted, multiple: false
+
+      work.property :date_modified, predicate: ::RDF::Vocab::DC.modified, multiple: false
+    end
+  end
+end

data/spec/support/shared_contexts/with_work_type.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+shared_context 'with a work type' do
+  # A work type must be defined for the default `RecordImporter` to save objects
+  before do
+    load './spec/support/hyrax/core_metadata.rb'
+    load './spec/support/hyrax/basic_metadata.rb'
+
+    class Work < ActiveFedora::Base
+      attr_accessor :visibility
+      attr_accessor :based_near_attributes
+      include ::Hyrax::CoreMetadata
+      include ::Hyrax::BasicMetadata
+    end
+
+    class User < Struct.new(:id, :user_key)
+      def initialize(inputs = {})
+        self.id = inputs[:id]
+        self.user_key = inputs[:user_key] || batch_user_key
+      end
+
+      def self.find_by_user_key(email)
+        User.new(user_key: email)
+      end
+
+      def self.find_or_create_system_user(_email)
+        User.new
+      end
+
+      def batch_user_key
+        'batchuser@example.com'
+      end
+    end
+
+    class Ability
+      def initialize(user); end
+    end
+
+    module Hyrax
+      def self.config
+        Config.new
+      end
+
+      class Config
+        def curation_concerns
+          [Work]
+        end
+      end
+
+      class Institution
+        def self.name
+          'my_institution'
+        end
+
+        def self.name_full
+          'my full institution name'
+        end
+      end
+
+      class UploadedFile < ActiveFedora::Base
+        def self.create(*)
+          h = Hyrax::UploadedFile.new
+          h.save
+          h
+        end
+      end
+
+      module Actors
+        class Environment
+          attr_reader :new_object, :attributes
+          def initialize(new_object, _ability, attributes)
+            @new_object = new_object
+            @attributes = attributes
+          end
+        end
+      end
+
+      class Actor
+        def create(actor_env)
+          attrs = actor_env.attributes
+          attrs.delete(:uploaded_files)
+          actor_env.new_object.attributes = attrs
+          actor_env.new_object.save
+        end
+      end
+
+      class CurationConcern
+        def self.actor
+          Hyrax::Actor.new
+        end
+      end
+    end
+  end
+
+  after do
+    Object.send(:remove_const, :Hyrax) if defined?(Hyrax)
+    Object.send(:remove_const, :Work) if defined?(Work)
+    Object.send(:remove_const, :User) if defined?(User)
+    Object.send(:remove_const, :Ability) if defined?(Ability)
+  end
+end

data/spec/zizia/csv_format_validator_spec.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Zizia::CsvFormatValidator do
+  subject(:validator)  { described_class.new(error_stream: []) }
+  let(:invalid_parser) { Zizia::CsvParser.new(file: invalid_file) }
+  let(:invalid_file)   { File.open('spec/fixtures/bad_example.csv') }
+
+  it_behaves_like 'a Zizia::Validator' do
+    let(:valid_parser)   { Zizia::CsvParser.new(file: valid_file) }
+    let(:valid_file)     { File.open('spec/fixtures/example.csv') }
+  end
+
+  define :a_validator_error do
+    match do |error|
+      return false unless error.respond_to?(:validator)
+
+      if fields
+        return false if fields[:validator] && error.validator != fields[:validator]
+        return false if fields[:name]      && error.name      != fields[:name]
+      end
+
+      true
+    end
+
+    chain :with, :fields
+  end
+
+  describe '#validate' do
+    it 'returns a Validator::Error' do
+      expect(validator.validate(parser: invalid_parser))
+        .to contain_exactly a_validator_error
+        .with(validator: validator.class,
+              name:      CSV::MalformedCSVError)
+    end
+  end
+end

data/spec/zizia/csv_parser_spec.rb

@@ -0,0 +1,73 @@
+# coding: utf-8
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'tempfile'
+
+describe Zizia::CsvParser do
+  subject(:parser) { described_class.new(file: file) }
+  let(:file)       { Tempfile.new(['fake', '.csv']) }
+
+  shared_context 'with content' do
+    let(:csv_content) do
+      <<-EOS
+title,description,date created
+The Moomins and the Great Flood,"The Moomins and the Great Flood (Swedish: Småtrollen och den stora översvämningen, literally The Little Trolls and the Great Flood) is a book written by Finnish author Tove Jansson in 1945, during the end of World War II. It was the first book to star the Moomins, but is often seen as a prelude to the main Moomin books, as most of the main characters are introduced in the next book.",1945
+Comet in Moominland,"Comet in Moominland is the second in Tove Jansson's series of Moomin books. Published in 1946, it marks the first appearance of several main characters, like Snufkin and the Snork Maiden.",1946
+EOS
+    end
+
+    let(:record_count) { 2 }
+
+    before do
+      file.write(csv_content)
+      file.rewind
+    end
+  end
+
+  it_behaves_like 'a Zizia::Parser' do
+    include_context 'with content'
+  end
+
+  it 'matches .csv files' do
+    expect(Zizia::Parser.for(file: file)).to be_a described_class
+  end
+
+  describe '#records' do
+    context 'with valid content' do
+      include_context 'with content'
+
+      it 'has the correct titles' do
+        expect(parser.records.map(&:title))
+          .to contain_exactly(['The Moomins and the Great Flood'],
+                              ['Comet in Moominland'])
+      end
+
+      it 'has correct other fields' do
+        expect(parser.records.map(&:date_created)).to contain_exactly(['1945'], ['1946'])
+      end
+    end
+
+    context 'with invalid file' do
+      let(:file) { File.open('spec/fixtures/bad_example.csv') }
+
+      it 'is empty' do
+        expect(parser.records.to_a).to be_empty
+      end
+    end
+  end
+
+  describe '#validate' do
+    it 'is valid' do
+      expect(parser.validate).to be_truthy
+    end
+
+    context 'with invalid file' do
+      let(:file) { File.open('spec/fixtures/bad_example.csv') }
+
+      it 'is invalid' do
+        expect(parser.validate).to be_falsey
+      end
+    end
+  end
+end

data/spec/zizia/formatted_message_stream_spec.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'zizia/streams/formatted_message_stream'
+
+describe Zizia::FormattedMessageStream do
+  subject(:stream)     { described_class.new(stream: fake_stream) }
+  let(:fake_stream)    { [] }
+
+  it_behaves_like 'a Zizia::MessageStream'
+
+  describe '#stream' do
+    subject(:stream) { described_class.new }
+
+    it 'is STDOUT by default' do
+      expect(stream.stream).to eq STDOUT
+    end
+  end
+
+  describe '#<<' do
+    it 'appends newlines by default' do
+      expect { stream << 'moomin' }
+        .to change { fake_stream }
+        .to contain_exactly("moomin\n")
+    end
+
+    it 'uses other % formatters' do
+      stream.formatter = "!!!%s!!!"
+
+      expect { stream << 'moomin' }
+        .to change { fake_stream }
+        .to contain_exactly('!!!moomin!!!')
+    end
+  end
+end

data/spec/zizia/hash_mapper_spec.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+describe Zizia::HashMapper do
+  it_behaves_like 'a Zizia::Mapper' do
+    let(:expected_fields) { metadata.keys.map(&:to_sym) }
+    let(:metadata) { { 'a_field' => 'a', 'b_field' => 'b', 'c_field' => 'c' } }
+  end
+end