hyrax-ingest 0.1.0

data/lib/hyrax/ingest/ingester/preservation_event_ingester.rb

@@ -0,0 +1,27 @@
+require 'hyrax/ingest/ingester/active_fedora_base_ingester'
+require 'hyrax/ingest/ingester/active_fedora_file_ingester'
+
+module Hyrax
+  module Ingest
+    module Ingester
+      class PreservationEventIngester < ActiveFedoraBaseIngester
+
+        attr_reader :premis_event_related_object
+
+        def initialize(config={})
+          config[:af_model_class_name] ||= 'Hyrax::Preservation::Event'
+          @premis_event_related_object = config.delete(:premis_event_related_object)
+          super(config)
+        end
+
+        def run!
+          assign_properties!
+          af_model.premis_event_related_object = premis_event_related_object
+          save_model!
+          # return the new instance of the ActiveFedora model
+          af_model
+        end
+      end
+    end
+  end
+end

data/lib/hyrax/ingest/ingester/work_ingester.rb

@@ -0,0 +1,55 @@
+require 'hyrax/ingest/ingester/active_fedora_base_ingester'
+require 'hyrax/ingest/ingester/file_set_ingester'
+
+module Hyrax
+  module Ingest
+    module Ingester
+      class WorkIngester < ActiveFedoraBaseIngester
+        attr_reader :file_sets_config
+
+        def initialize(config={})
+          # TODO: Throw a useful custom error when :type option is missing.
+          config[:af_model_class_name] ||= config.delete(:type)
+          @file_sets_config = config.delete(:FileSets) || []
+          super(config)
+        end
+
+        def run!
+          assign_properties!
+          assign_related_file_set_properties!
+          apply_depositor_metadata!
+          save_model!
+          assign_file_sets_to_work!
+          # return the new instance of the ActiveFedora model
+          af_model
+        end
+
+        private
+
+          def apply_depositor_metadata!
+            af_model.apply_depositor_metadata(depositor) if depositor
+          end
+
+          def assign_related_file_set_properties!
+            file_set_ingesters.each { |file_set_ingester| file_set_ingester.assign_properties! }
+          end
+
+          def assign_file_sets_to_work!
+            file_set_ingesters.each do |file_set_ingester|
+              af_model.members += [file_set_ingester.af_model]
+            end
+          end
+
+          def file_set_ingesters
+            @file_set_ingesters ||= @file_sets_config.map do |file_set_config|
+              Hyrax::Ingest::Ingester::FileSetIngester.new(file_set_config).tap do |file_set_ingester|
+                file_set_ingester.sip = sip if file_set_ingester.respond_to?(:sip=)
+                file_set_ingester.shared_sip = shared_sip if file_set_ingester.respond_to?(:shared_sip=)
+                file_set_ingester.iteration = iteration if file_set_ingester.respond_to?(:iteration=)
+              end
+            end
+          end
+      end
+    end
+  end
+end

data/lib/hyrax/ingest/reporting.rb

@@ -0,0 +1,13 @@
+require 'hyrax/ingest/reporting/configuration'
+
+module Hyrax
+  module Ingest
+    module Reporting
+      class << self
+        def config
+          @config ||= Hyrax::Ingest::Reporting::Configuration.new
+        end
+      end
+    end
+  end
+end

data/lib/hyrax/ingest/reporting/configuration.rb

@@ -0,0 +1,22 @@
+require 'hyrax/ingest/errors'
+
+module Hyrax
+  module Ingest
+    module Reporting
+      class Configuration
+        attr_reader :default_template_path
+        attr_accessor :default_output_file
+
+        def initialize
+          @default_template_path = File.expand_path('../views/hyrax_ingest_report.html.erb', __FILE__)
+          @default_output_file = File.expand_path('hyrax_ingest_report.html')
+        end
+
+        def default_template_path=(path)
+          raise Hyrax::Ingest::Errors::ConfigurationError, "\"#{path}\" does not exist" unless File.exist? path
+          @default_template_path = path
+        end
+      end
+    end
+  end
+end

data/lib/hyrax/ingest/reporting/report.rb

@@ -0,0 +1,79 @@
+require 'erb'
+require 'functional_hash'
+require 'hyrax/ingest/reporting/configuration'
+
+module Hyrax
+  module Ingest
+    module Reporting
+      class Report
+        attr_reader :stat
+
+        def initialize
+          @stat = default_stat
+        end
+
+        def default_stat
+          # Initialize a FunctionalHash to serve as a stat tracker, and
+          # add some default values in the same way you would a Hash.
+          @stat = FunctionalHash.new.tap do |stat|
+            # Stores an array of all SIP paths.
+            stat[:sip_paths] = []
+
+            # Stores a list of all files from SIPs that are part of the ingest.
+            stat[:files] = []
+
+            # Stores a hash where keys are Fetcher classes, and values are
+            # the number of occurrences of missing rquired values.
+            stat[:missing_required_values] = {}
+
+            stat[:total_missing_required_values] = Proc.new do |s|
+              stat[:missing_required_values].reduce(0) do |total, key_and_value|
+                # When reducing a Hash, the 2nd arg to the block is a
+                # 2-element array, where the 1st element is the key, and the
+                # 2nd element is the value the key points to.
+                occurrences = key_and_value.last
+                total + occurrences.count
+              end
+            end
+
+            # Filters the :missing_required_values hash to those for XML files.
+            # stat[:xml_files_missing_required_values] = Proc.new do |s|
+            #   s[:missing_required_values].select { |fetcher_class, params| fetcher_class.to_s =~ /XMLFile$/ }
+            # end
+
+            # Define a functional hash value that returns the count of the given key.
+            stat[:count] = Proc.new do |s, key_to_count|
+              s[key_to_count].respond_to?(:count) ? s[key_to_count].count : 0
+            end
+
+            stat[:models_saved] = []
+            stat[:models_failed] = []
+          end
+        end
+
+        def render(template_path: nil)
+          template_path ||= Reporting.config.default_template_path
+          template_content = File.read(File.expand_path(template_path))
+          ERB.new(template_content).result(binding)
+        end
+
+        def write_to_file(filename: nil, template_path: nil)
+          filename ||= Reporting.config.default_output_file
+          File.write(filename, render(template_path: template_path))
+        end
+
+        def failed_with(error)
+          errors << error
+        end
+
+        def errors
+          @errors ||= []
+        end
+
+        def failed?
+          !errors.empty?
+        end
+      end
+    end
+  end
+end

data/lib/hyrax/ingest/reporting/views/hyrax_ingest_report.html.erb

@@ -0,0 +1,77 @@
+<style>
+
+dl.inline dd {
+  display: inline;
+  margin: 0;
+}
+
+dl.inline dd:after {
+  display: block;
+  content: '';
+}
+
+dl.inline dt {
+  display: inline-block;
+  /*min-width: 100px;*/
+  width: 200px;
+}
+
+h3 {
+  margin-left: 20px;
+}
+
+</style>
+
+<h1>Hyrax Ingest Report Summary</h1>
+
+<% if failed? %>
+  <h2><strong>Ingest Failed!<strong></h2>
+  <p>The following errors caused the ingest to fail:</p>
+  <ol>
+    <% errors.each do |error| %>
+      <li><%= error %></li>
+    <% end %>
+  </ol>
+<% end %>
+
+
+<h2>Ingest Details:</h2>
+<dl class="inline">
+  <dt>Ingest configuration file:</dt>
+  <dd><%= stat[:config_file_path] %></dd>
+  <dt>Started at:</dt>
+  <dd><%= stat[:datetime_started] %></dd>
+  <dt>Completed at:</dt>
+  <dd><%= stat[:datetime_completed] %></dd>
+  <dt>Batch size:</dt>
+  <dd><%= stat[:batch_size] %></dd>
+  <dt>Total # of source files used:</dt>
+  <dd><%= stat[:count, :files] %></dd>
+  <dt>Records ingested:</dt>
+  <dd><%= stat[:count, :models_saved] %></dd>
+  <dt>Records failed:</dt>
+  <dd><%= stat[:count, :models_failed] %></dd>
+</dl>
+
+<h2>Missing Required Values: <%= stat[:total_missing_required_values] %> total</h2>
+<% if stat[:total_missing_required_values] > 0 %>
+  <% stat[:missing_required_values].each do |fetcher_class, list_of_param_hashes| %>
+    <% fetcher_class_short_name = fetcher_class.to_s.gsub(/.*\:\:/, '') %>
+    <h3>Missing required values from <%= fetcher_class_short_name %>: <%= list_of_param_hashes.count %></h3>
+    <ol>
+      <% list_of_param_hashes.each do |param_hash| %>
+        <li>
+          <% if param_hash.empty? %>
+            <em>No additional information</em>
+          <% else %>
+            <dl class="inline">
+              <% param_hash.each do |key, val| %>
+                <dt><%= key%>:</dt><dd><%= val %></dd>
+              <% end %>
+            </dl>
+          <% end %>
+        </li>
+      <% end%>
+    </ol>
+  <% end %>
+<% end %>

data/lib/hyrax/ingest/reporting/views/hyrax_ingest_report.txt.erb

@@ -0,0 +1,21 @@
+=============================================
+======== Hyrax Ingest Report Summary ========
+=============================================
+
+Ingest configuration file:      <%= stat[:config_file_path] %>
+Started at:                     <%= stat[:datetime_started] %>
+Completed at:                   <%= stat[:datetime_completed] %>
+Batch size:                     <%= stat[:batch_size] %>
+Total # of source files used:   <%= stat[:count, :files] %>
+Records ingested:               <%= stat[:count, :models_saved] %>
+Records failed:                 <%= stat[:count, :models_failed] %>
+
+<% if stat[:count, :xml_files_missing_required_values] %>
+XML Files missing required values...
+  <% stat[:xml_files_missing_required_values].each do |filename, xpaths| %>
+  Values missing from: <%= filename %>
+    <% xpaths.each do |xpath| %>
+    - <%= xpath %>
+    <% end %>
+  <% end %>
+<% end %>

data/lib/hyrax/ingest/runner.rb

@@ -0,0 +1,103 @@
+require 'hyrax/ingest/configuration'
+require 'hyrax/ingest/ingester'
+require 'hyrax/ingest/reporting'
+require 'hyrax/ingest/has_sip'
+require 'hyrax/ingest/has_shared_sip'
+require 'hyrax/ingest/has_iteration'
+require 'hyrax/ingest/has_logger'
+require 'hyrax/ingest/has_report'
+require 'hyrax/ingest/has_depositor'
+
+
+module Hyrax
+  module Ingest
+    class Runner
+      include Reporting
+      include Interloper
+      include HasSIP
+      include HasSharedSIP
+      include HasIteration
+      include HasReport
+      include HasLogger
+      include HasDepositor
+
+      attr_reader :config
+
+      before(:run!) do
+        logger.info "Ingest iteration #{iteration+1} started."
+        report.stat[:datetime_started] ||= DateTime.now
+        report.stat[:batch_size] ||= 1
+        report.stat[:files] += sip.file_paths if sip
+        report.stat[:files] += shared_sip.file_paths if shared_sip
+        report.stat[:config_file_path] = config.config_file_path
+      end
+
+      after(:run!) do
+        logger.info "Ingest iteration #{iteration+1} complete."
+        report.stat[:datetime_completed] ||= DateTime.now
+      end
+
+      def initialize(config_file_path:, sip_path: nil, shared_sip_path: nil, iteration: 0, depositor: nil)
+        self.sip = SIP.new(path: sip_path) if sip_path
+        self.shared_sip = shared_sip_path != nil ? SIP.new(path: shared_sip_path) : nil
+        self.iteration = iteration.to_i
+        self.depositor = depositor
+        @config = Hyrax::Ingest::Configuration.new(config_file_path: config_file_path)
+      end
+
+      def run!
+        ingesters.collect { |ingester| ingester.run! }
+      end
+
+      # TODO: Does not yet return IDs of associated objects that were ingested
+      # as assocaited objects (i.e. objects that are nested under other
+      # objects in the ingest configuration). It only returns IDs for objects that
+      # are ingested per the top-level of ingest configuration.
+      def ingested_ids_by_type
+        {}.tap do |h|
+          ingesters.each do |ingester|
+            if ingester.respond_to? :af_model
+              h[ingester.af_model.class] ||= []
+              h[ingester.af_model.class] << ingester.af_model.id
+            end
+          end
+        end
+      end
+
+      # TODO: Does not yet return IDs of associated objects that were ingested
+      # as assocaited objects (i.e. objects that are nested under other
+      # objects in the ingest configuration). It only returns IDs for objects that
+      # are ingested per the top-level of ingest configuration.
+      def ingested_ids_by_type
+        {}.tap do |h|
+          ingesters.each do |ingester|
+            if ingester.respond_to? :af_model
+              h[ingester.af_model.class] ||= []
+              h[ingester.af_model.class] << ingester.af_model.id
+            end
+          end
+        end
+      end
+
+      private
+
+        def ingesters
+          @ingesters ||= config.ingester_configs.map do |ingester_config|
+            # TODO: Better way to handle invalid config than throwing big
+            # error msgs from here.
+            raise Hyrax::Ingest::Errors::InvalidConfig.new('Ingester config must be a single key value pair, where the key is the name of the ingester, and the value is the ingester configuration.') unless ingester_config.respond_to? :keys
+            ingester_name = ingester_config.keys.first
+            ingester_options = ingester_config.values.first
+            Hyrax::Ingest::Ingester.factory(ingester_name, ingester_options).tap do |ingester|
+              ingester.sip = sip if ingester.respond_to? :sip=
+              ingester.shared_sip = shared_sip if ingester.respond_to? :shared_sip=
+              ingester.iteration = iteration if ingester.respond_to? :iteration=
+              ingester.logger = logger if ingester.respond_to? :logger=
+              ingester.report = report if ingester.respond_to? :report=
+              ingester.depositor = depositor if ingester.respond_to? :depositor=
+            end
+          end
+        end
+    end
+  end
+end

data/lib/hyrax/ingest/sip.rb

@@ -0,0 +1,92 @@
+require 'hyrax/ingest/errors'
+require 'minitar'
+
+module Hyrax
+  module Ingest
+    # A model for reading Submission Information Packages (SIPs) from a file_pathsystem.
+    #
+    # @attr_reader [String] path description of a readonly attribute
+    class SIP
+      attr_reader :path
+
+      # @param [String] path The path to the SIP on the filesystem.
+      def initialize(path:)
+        raise Hyrax::Ingest::Errors::InvalidSIPPath.new(path.to_s) unless File.exists? path.to_s
+        @path = path
+      end
+
+      # @return [Array] A list of File objects that are part of the SIP
+      def file_paths
+        @file_paths ||= single_file_path
+        @file_paths ||= file_paths_from_dir
+        @file_paths ||= file_paths_from_tarball
+        @file_paths ||= []
+      end
+
+      # @param [String, Regexp] filename A string, a Regexp, or a string representation of a regex
+      # @return [File] The file from the SIP that matches the param.
+      def find_file_path(basename_or_regex)
+        file_path = file_path_from_regex(basename_or_regex) || file_path_from_basename(basename_or_regex)
+        raise Hyrax::Ingest::Errors::FileNotFoundInSIP.new(path, basename_or_regex) unless file_path
+        file_path
+      end
+
+      # Reads the content of a file from the SIP, and automatically rewinds it
+      # so it can be read again.
+      # @param [String, Regexp] filename A string, a Regexp, or a string representation of a regex
+      # @return [String] The contents of the matched file
+      def read_file(basename_or_regex)
+        File.read(find_file_path(basename_or_regex))
+      end
+
+      private
+
+        # @param [String, Regexp] regex Either a Regexp object or a string
+        #   beginning and ending in forward slashes, that can be converted to
+        #   a regex.
+        # @return [File] The file that matches regex as a regular expression;
+        #   nil if no file matches 'regex', or if 'regex' cannot be used as a
+        #   regular expression.
+        def file_path_from_regex(regex)
+          # If 'regex' is a string beginning and ending in slash, convert it to
+          # a Regexp.
+          regex = Regexp.new(regex.to_s[1..-2]) if regex.to_s =~ /^\/.*\/$/
+          file_paths.find { |file| File.basename(file) =~ regex } if regex.is_a? Regexp
+        end
+
+        # @param [String] filename The name of the file within the SIP you want
+        #   to return.
+        # @return [File] The file that matches the 'filename' parameter; nil if
+        #   no file matches the 'filename'.
+        def file_path_from_basename(filename)
+          file_paths.find { |file| File.basename(file) == filename }
+        end
+
+        # @return Array An Array containing the one and only file pointed to by #path
+        def single_file_path
+          Array(path) if File.file? path
+        end
+
+        def file_paths_from_dir
+          if File.directory? path
+            Dir.glob("#{path}/**/*").select { |entry| File.file? entry }
+          end
+        end
+
+        def file_paths_from_tarball
+          # TODO: this is the best test I could find for reliably determining
+          # whether a file was a TAR archive or not, but it seems finicky, as
+          # it probably depends on your operating system, or what kind of tarball
+          # it is. Find something better?
+          if (`file '#{path}'` =~ /tar archive/)
+            Minitar.unpack(path, tmp_dir_for_unpacked_tarball)
+            Dir.glob("#{tmp_dir_for_unpacked_tarball}/**/*")
+          end
+        end
+
+        def tmp_dir_for_unpacked_tarball
+          "#{Dir.tmpdir}/#{File.basename(path)}.unpacked"
+        end
+    end
+  end
+end