hyrax-ingest 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: c0d5eeb006d3931a1a7668b71e9a0ce85d603bbff3025aba8552eb42958806d4
+  data.tar.gz: 97de1d90af9b90972ceb24df28ffebd9687d9ac63fc18fc4d59ec96e54359949
+SHA512:
+  metadata.gz: 1175542851df962db08f1e1568ff5d07a53a9630995d9af0ff1b0c6a7d7cff08a871e4c03d28f479998198bd6e2f8b990874510f8424259440ae21e468429c0c
+  data.tar.gz: 12ffb343ec907eefe6c8ac367e8bb963239a32f2aea789189e9e11ed21c8eacb27983d8365adf42505531ff523995b79701d998425f907290d6b74467acb99c8

data/README.md

@@ -0,0 +1,52 @@
+# Hyrax::Ingest
+
+[![Build Status](https://travis-ci.org/IUBLibTech/hyrax-ingest.svg?branch=master)](https://travis-ci.org/IUBLibTech/hyrax-ingest)
+
+Hyrax::Ingest is an extensible plugin for ingesting content and metadata into
+a Hyrax[https://github.com/samvera/hyrax] repository based on a declarative
+configuration.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'hyrax-ingest'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install hyrax-ingest
+
+## Overview
+
+The Hyrax::Ingest plugin defines "ingest" as the process of mapping content
+and data from one or more sources to one or more persistence layers to be used
+by a Hyrax application.
+
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then,
+run `rake spec` to run the tests. You can also run `bin/console` for an
+interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`.
+To release a new version, update the version number in `version.rb`, and then
+run `bundle exec rake release`, which will create a git tag for the version,
+push git commits and tags, and push the `.gem` file to
+[rubygems.org](https://rubygems.org).
+
+## Contributing
+
+TODO: Add CONTRIBUTING.md per the Samvera standard, and refer to it here.
+
+## License
+
+The gem is available as open source under the terms of the [MIT
+License](http://opensource.org/licenses/MIT).
+

data/config/routes.rb

@@ -0,0 +1,2 @@
+Rails.application.routes.draw do
+end

data/lib/hyrax/ingest.rb

@@ -0,0 +1,12 @@
+require 'hyrax/ingest/engine'
+require 'hyrax'
+
+module Hyrax
+  module Ingest
+    class << self
+      def root
+        File.expand_path('../../../', __FILE__)
+      end
+    end
+  end
+end

data/lib/hyrax/ingest/batch_runner.rb

@@ -0,0 +1,130 @@
+require 'hyrax/ingest/runner'
+require 'hyrax/ingest/has_report'
+require 'hyrax/ingest/has_logger'
+require 'hyrax/ingest/has_depositor'
+require 'interloper'
+
+module Hyrax
+  module Ingest
+    class BatchRunner
+      include HasReport
+      include HasLogger
+      include HasDepositor
+      include Interloper
+
+      # Report before and after a batch is ingested.
+      before(:run!) do
+        logger.info "Starting batch ingest, batch size = #{iterations}."
+        report.stat[:batch_size] = iterations
+        report.stat[:datetime_started] = DateTime.now
+      end
+
+      after(:run!) do
+        logger.info "Batch ingest complete."
+        report.stat[:datetime_completed] = DateTime.now
+        report.write_to_file
+      end
+
+      attr_reader :sip_paths
+
+      def initialize(config_file_path:, sip_paths: [], shared_sip_path: nil, iterations: nil, depositor: nil)
+        @sip_paths = sip_paths
+        @shared_sip_path = shared_sip_path
+        @config_file_path = config_file_path
+        @iterations = iterations
+        self.depositor = depositor
+      end
+
+      def run!
+        runners.each { |runner| runner.run! }
+      rescue StandardError => error
+        # TODO: move to callback, but :rescue hook doesn't exist yet in
+        # Interloper gem.
+        report.stat[:datetime_completed] = DateTime.now
+        report.failed_with error
+        raise error
+      ensure
+        # TODO: move to callback, but :ensure hook doesn't exist yet in
+        # Interloper gem.
+        report.write_to_file
+      end
+
+      # Returns an array containing the IDs of new or updated records.
+      # Currently only returns the IDs for ActiveFedora records (or
+      # subclasses) that are specified at the top level (i.e. not nested) of
+      # the ingest configuration.
+      # @return [Array] list of IDs
+      def ingested_ids
+        ingested_ids_by_type.flatten
+      end
+
+      # Returns an hash containing the IDs of new or updated records, keyed by
+      # the model class by which they were saved.
+      # Example:
+      #   { FileSet => ['123', '456'], Work => ['789'] }
+      # Currently only returns the IDs for ActiveFedora records (or
+      # subclasses) that are specified at the top level (i.e. not nested) of
+      # the ingest configuration.
+      # @return [Hash] IDs keyed by the model class by which they were saved.
+      def ingested_ids_by_type
+        {}.tap do |h|
+          runners.each do |runner|
+            runner.ingested_ids_by_type.each do |type, ids|
+              h[type] ||= []
+              h[type] += ids
+              h[type].uniq!
+            end
+          end
+        end
+      end
+
+      # Returns an array containing the IDs of new or updated records.
+      # Currently only returns the IDs for ActiveFedora records (or
+      # subclasses) that are specified at the top level (i.e. not nested) of
+      # the ingest configuration.
+      # @return [Array] list of IDs
+      def ingested_ids
+        ingested_ids_by_type.flatten
+      end
+
+      # Returns an hash containing the IDs of new or updated records, keyed by
+      # the model class by which they were saved.
+      # Example:
+      #   { FileSet => ['123', '456'], Work => ['789'] }
+      # Currently only returns the IDs for ActiveFedora records (or
+      # subclasses) that are specified at the top level (i.e. not nested) of
+      # the ingest configuration.
+      # @return [Hash] IDs keyed by the model class by which they were saved.
+      def ingested_ids_by_type
+        {}.tap do |h|
+          runners.each do |runner|
+            runner.ingested_ids_by_type.each do |type, ids|
+              h[type] ||= []
+              h[type] += ids
+              h[type].uniq!
+            end
+          end
+        end
+      end
+
+      private
+
+        def iterations
+          # Return @iterations.to_i if it's not nil and not empty
+          unless @iterations.nil? || @iterations.to_s.empty?
+            @iterations
+          else
+            [1, @sip_paths.count].max
+          end.to_i
+        end
+
+        def runners
+          @runners ||= (0...iterations).map do |iteration|
+            Hyrax::Ingest::Runner.new(config_file_path: @config_file_path, sip_path: @sip_paths[iteration], shared_sip_path: @shared_sip_path, iteration: iteration, depositor: depositor).tap do |runner|
+              runner.report = report if runner.respond_to? :report=
+            end
+          end
+        end
+    end
+  end
+end

data/lib/hyrax/ingest/configuration.rb

@@ -0,0 +1,54 @@
+require 'hyrax/ingest/errors'
+require 'active_support/core_ext/hash/keys'
+
+module Hyrax
+  module Ingest
+    class Configuration
+      attr_reader :config_file_path
+
+      def initialize(config_file_path:)
+        @config_file_path = config_file_path.to_s
+        raise Hyrax::Ingest::Errors::NoConfigFileFound.new(@config_file_path) unless File.exist? @config_file_path
+        validate!
+      end
+
+      # @return [Array] Array of hashes, where each hash is the configuration options
+      #   for an Ingester instance
+      def ingester_configs
+        @ingester_configs ||= config[:ingest]
+      end
+
+      private
+
+        # @return [Hash] The config hash parsed from yaml file, and with keys
+        #   converted from strings to symbols.
+        def config
+          @config ||= Psych.load_file(config_file_path).deep_symbolize_keys
+        end
+
+        # Validates the configuration.
+        # @raise [Hyrax::Ingest::Errors::InvalidConfig] When the configuration is invalid.
+        def validate!
+          validate_top_level_key!
+          validate_ingester_configs_array!
+          validate_ingester_config_hashes!
+        end
+
+        # @raise [Hyrax::Ingest::Errors::InvalidConfig] When the top level
+        #   'ingest' key is missing.
+        def validate_top_level_key!
+          raise Hyrax::Ingest::Errors::InvalidConfig.new(config_file_path, "Top-level key 'ingest' is missing.") unless config[:ingest]
+        end
+
+        def validate_ingester_configs_array!
+          raise Hyrax::Ingest::Errors::InvalidConfig.new(config_file_path, "Value under top-level 'ingest' key must be an array containing the configuration for each ingester you want to use.") unless config[:ingest].respond_to?(:each)
+        end
+
+        def validate_ingester_config_hashes!
+          config[:ingest].each do |ingest_config|
+            raise Hyrax::Ingest::Errors::InvalidConfig.new(config_file_path, "Each ingester configuration must be a single key-value pair, where the key is the type of ingester, and the value is a hash containing the configuration for the ingester. But a #{ingester_config.class} was found instead.") unless ingest_config.respond_to? :keys
+          end
+        end
+    end
+  end
+end

data/lib/hyrax/ingest/engine.rb

@@ -0,0 +1,6 @@
+module Hyrax
+  module Ingest
+    class Engine < ::Rails::Engine
+    end
+  end
+end

data/lib/hyrax/ingest/errors.rb

@@ -0,0 +1,186 @@
+module Hyrax
+  module Ingest
+    # Base class that allows for rescuing from all Hyrax::Ingest errors.
+    # Intentionally no-op.
+    class Error < StandardError; end
+
+    module Errors
+      class InvalidSIPPath < Hyrax::Ingest::Error
+        def initialize(invalid_path)
+          super("#{invalid_path} is not a valid SIP path")
+        end
+      end
+
+      class NoSIPSpecified < Hyrax::Ingest::Error
+        def initialize(obj)
+          super("No SIP was specified.")
+        end
+      end
+
+      class NoSharedSIPSpecified < Hyrax::Ingest::Error
+        def initialize
+          super("No shared SIP was specified.")
+        end
+      end
+
+      class InvalidSIP < Hyrax::Ingest::Error
+        def initialize(invalid_sip_obj)
+          super("Invalid SIP object. SIP must be an instance of Hyrax::Ingest::SIP (or an instance of a subclass) but an instance of #{invalid_sip_obj.class} was found.")
+        end
+      end
+
+      class UnknownActiveFedoraModel < Hyrax::Ingest::Error
+        def initialize(af_model_class_name)
+          super("Unknown ActiveFedora model type '#{af_model_class_name.to_s}'")
+        end
+      end
+
+      class FileNotFoundInSIP < Hyrax::Ingest::Error
+        def initialize(sip_path, string_or_regexp)
+          super("No file matching #{string_or_regexp.inspect.to_s} was found in the SIP at path '#{sip_path}'")
+        end
+      end
+
+      class UnknownAssignerClass < Hyrax::Ingest::Error
+        def initialize(class_name, available_classes)
+          super("Assigner class '#{class_name}' not found. Available assigner classes are: #{Array(available_classes).join(', ')}")
+        end
+      end
+
+      class AmbiguousAssignerClass < Hyrax::Ingest::Error
+        def initialize(class_name, matching_classes)
+          super("Assigner class name '#{class_name}' is ambiguous; could mean any one of the following classes: #{Array(matching_classes).join(',')}. Please use namespaces in the class name to be more specific.")
+        end
+      end
+
+      class UnknownFetcherClass < Hyrax::Ingest::Error
+        def initialize(class_name, available_classes)
+          super("Fetcher class '#{class_name}' not found. Available transformer classes are: #{Array(available_classes).join(', ')}")
+        end
+      end
+
+      class AmbiguousFetcherClass < Hyrax::Ingest::Error
+        def initialize(class_name, matching_classes)
+          super("Fetcher class name '#{class_name}' is ambiguous; could mean any one of the following classes: #{Array(matching_classes).join(',')}. Please use namespaces in the class name to be more specific.")
+        end
+      end
+
+      class InvalidFetcher < Hyrax::Ingest::Error
+        def initialize(class_name)
+          super("Invalid fetcher '#{class_name}'; Fetcher objects must extend Hyrax::Ingest::Fetcher::Base.")
+        end
+      end
+
+      class MissingConfigOptions < Hyrax::Ingest::Error
+        def initialize(config_options)
+          super("Missing config options: '#{config_options.join(', ')}'")
+        end
+      end
+
+      class UnknownIngesterClass < Hyrax::Ingest::Error
+        def initialize(class_name, available_classes)
+          super("Ingester class '#{class_name}' not found. Available ingester classes are: #{Array(available_classes).join(', ')}")
+        end
+      end
+
+      class AmbiguousIngesterClass < Hyrax::Ingest::Error
+        def initialize(class_name, matching_classes)
+          super("Ingester class name '#{class_name}' is ambiguous; could mean any one of the following classes: #{Array(matching_classes).join(',')}. Please use namespaces in the class name to be more specific.")
+        end
+      end
+
+      class RecordNotFound < Hyrax::Ingest::Error
+        def initialize(model_class_name, where_clause)
+          super("Record of type '#{model_class_name}' could not be found where #{where_clause}.")
+        end
+      end
+
+      class AmbiguousFetchOptions < Hyrax::Ingest::Error
+        def initialize(ambiguous_options)
+          super("Could not determine which transformer class to use given the following options: #{Array(ambiguous_options).join(', ')}")
+        end
+      end
+
+      class MissingRequiredFetchOptions < Hyrax::Ingest::Error
+        def initialize(missing_options)
+          super("Missing required assignment options: #{Array(missing_options).join(', ')}")
+        end
+      end
+
+      class InvalidFetchOptions < Hyrax::Ingest::Error
+        def initialize(invalid_options)
+          super("Invalid fetch options: #{Array(invalid_options).join(', ')}")
+        end
+      end
+
+      class InvalidConfig < Hyrax::Ingest::Error
+        def initialize(config_file_path, msg=nil)
+          message = ["Invalid configuration in '#{config_file_path}'.", msg.to_s].join("\n")
+          super(message)
+        end
+      end
+
+      class InvalidIngesterClass < Hyrax::Ingest::Error
+        def initialize(invalid_class)
+          super("Invalid ingester class #{invalid_class} does not extend Hyrax::Ingest::Ingester::Base")
+        end
+      end
+
+      class UnknownTransformerClass < Hyrax::Ingest::Error
+        def initialize(class_name, available_classes)
+          super("Transformer class '#{class_name}' not found. Available transformer classes are: #{Array(available_classes).join(', ')}")
+        end
+      end
+
+      class AmbiguousTransformerClass < Hyrax::Ingest::Error
+        def initialize(class_name, matching_classes)
+          super("Transformer class name '#{class_name}' is ambiguous; could mean any one of the following classes: #{Array(matching_classes).join(',')}. Please use namespaces in the class name to be more specific.")
+        end
+      end
+
+      class NoConfigFileFound < Hyrax::Ingest::Error
+        def initialize(path)
+          super("No ingest config file exists at '#{path}'.")
+        end
+      end
+
+      class UnknownRdfPredicate < Hyrax::Ingest::Error
+        def initialize(unknown_rdf_predicate, af_model_class_name)
+          super("Unknown RDF Predicate '#{unknown_rdf_predicate}' for ActiveFedora model '#{af_model_class_name}'")
+        end
+      end
+
+      class UnrecognizedTransformOption < Hyrax::Ingest::Error
+        def initialize(unrecognized_transform_option)
+          super("Unknown transform option: '#{unrecognized_transform_option}'")
+        end
+      end
+
+      class UnableToPrintIngestReport < Hyrax::Ingest::Error
+        def initialize(unable_to_read_from_this_thing)
+          super("Cannot print ingest report from #{unable_to_read_from_this_thing}. To print a report, it must be written to a file.")
+        end
+      end
+
+      class InvalidReportConfigOptions < Hyrax::Ingest::Error
+        def initialize(invalid_options)
+          super("Invalid reporting config options: #{invalid_options.join(', ')}")
+        end
+      end
+
+      class MissingCsvColumn < Hyrax::Ingest::Error
+        def initialize(invalid_column)
+          super("Unknown column header for: #{invalid_column}.")
+        end
+      end
+
+      class ConfigurationError < Hyrax::Ingest::Error; end
+
+      class InvalidActiveFedoraPropertyValue < Hyrax::Ingest::Error
+        def initialize(value, property_name, rdf_predicate)
+          super("Could not assign '#{value}' to property #{property_name} (with RDF predicate '#{rdf_predicate}')")
+        end
+      end
+    end
+  end
+end