zizia 1.0.1

data/lib/zizia/hyrax_record_importer.rb

@@ -0,0 +1,261 @@
+# frozen_string_literal: true
+
+module Zizia
+  class HyraxRecordImporter < RecordImporter
+    # TODO: Get this from Hyrax config
+    DEFAULT_CREATOR_KEY = 'batchuser@example.com'
+
+    # @!attribute [rw] depositor
+    # @return [User]
+    attr_accessor :depositor
+
+    # @!attribute [rw] collection_id
+    # @return [String] The fedora ID for a Collection.
+    attr_accessor :collection_id
+
+    # @!attribute [rw] batch_id
+    # @return [String] an id number associated with the process that kicked off this import run
+    attr_accessor :batch_id
+
+    # @!attribute [rw] deduplication_field
+    # @return [String] if this is set, look for records with a match in this field
+    # and update the metadata instead of creating a new record. This will NOT re-import file attachments.
+    attr_accessor :deduplication_field
+
+    # @!attribute [rw] success_count
+    # @return [String] the number of records this importer has successfully created
+    attr_accessor :success_count
+
+    # @!attribute [rw] failure_count
+    # @return [String] the number of records this importer has failed to create
+    attr_accessor :failure_count
+
+    # @param attributes [Hash] Attributes that come
+    #        from the UI or importer rather than from
+    #        the CSV/mapper. These are useful for logging
+    #        and tracking the output of an import job for
+    #        a given collection, user, or batch.
+    #        If a deduplication_field is provided, the system will
+    #        look for existing works with that field and matching
+    #        value and will update the record instead of creating a new record.
+    # @example
+    #   attributes: { collection_id: '123',
+    #                 depositor_id: '456',
+    #                 batch_id: '789',
+    #                 deduplication_field: 'legacy_id'
+    #               }
+    def initialize(error_stream: Zizia.config.default_error_stream,
+                   info_stream: Zizia.config.default_info_stream,
+                   attributes: {})
+      self.collection_id = attributes[:collection_id]
+      self.batch_id = attributes[:batch_id]
+      self.deduplication_field = attributes[:deduplication_field]
+      set_depositor(attributes[:depositor_id])
+      @success_count = 0
+      @failure_count = 0
+      super(error_stream: error_stream, info_stream: info_stream)
+    end
+
+    # "depositor" is a required field for Hyrax.  If
+    # it hasn't been set, set it to the Hyrax default
+    # batch user.
+    def set_depositor(user_key)
+      user = ::User.find_by_user_key(user_key) if user_key
+      user ||= ::User.find(user_key) if user_key
+      user ||= ::User.find_or_create_system_user(DEFAULT_CREATOR_KEY)
+      self.depositor = user
+    end
+
+    ##
+    # @param record [ImportRecord]
+    # @return [ActiveFedora::Base]
+    # Search for any existing records that match on the deduplication_field
+    def find_existing_record(record)
+      return unless deduplication_field
+      return unless record.respond_to?(deduplication_field)
+      return if record.mapper.send(deduplication_field).nil?
+      return if record.mapper.send(deduplication_field).empty?
+      existing_records = import_type.where("#{deduplication_field}": record.mapper.send(deduplication_field).to_s)
+      raise "More than one record matches deduplication_field #{deduplication_field} with value #{record.mapper.send(deduplication_field)}" if existing_records.count > 1
+      existing_records&.first
+    end
+
+    ##
+    # @param record [ImportRecord]
+    #
+    # @return [void]
+    def import(record:)
+      existing_record = find_existing_record(record)
+      create_for(record: record) unless existing_record
+      update_for(existing_record: existing_record, update_record: record) if existing_record
+    rescue Faraday::ConnectionFailed, Ldp::HttpError => e
+      error_stream << e
+    rescue RuntimeError => e
+      error_stream << e
+      raise e
+    end
+
+    # TODO: You should be able to specify the import type in the import
+    def import_type
+      raise 'No curation_concern found for import' unless
+        defined?(Hyrax) && Hyrax&.config&.curation_concerns&.any?
+
+      Hyrax.config.curation_concerns.first
+    end
+
+    # The path on disk where file attachments can be found
+    def file_attachments_path
+      ENV['IMPORT_PATH'] || '/opt/data'
+    end
+
+    # Create a Hyrax::UploadedFile for each file attachment
+    # TODO: What if we can't find the file?
+    # TODO: How do we specify where the files can be found?
+    # @param [Zizia::InputRecord]
+    # @return [Array] an array of Hyrax::UploadedFile ids
+    def create_upload_files(record)
+      return unless record.mapper.respond_to?(:files)
+      files_to_attach = record.mapper.files
+      return [] if files_to_attach.nil? || files_to_attach.empty?
+
+      uploaded_file_ids = []
+      files_to_attach.each do |filename|
+        file = File.open(find_file_path(filename))
+        uploaded_file = Hyrax::UploadedFile.create(user: @depositor, file: file)
+        uploaded_file_ids << uploaded_file.id
+        file.close
+      end
+      uploaded_file_ids
+    end
+
+    ##
+    # Within the directory specified by ENV['IMPORT_PATH'], find the first
+    # instance of a file matching the given filename.
+    # If there is no matching file, raise an exception.
+    # @param [String] filename
+    # @return [String] a full pathname to the found file
+    def find_file_path(filename)
+      filepath = Dir.glob("#{ENV['IMPORT_PATH']}/**/#{filename}").first
+      raise "Cannot find file #{filename}... Are you sure it has been uploaded and that the filename matches?" if filepath.nil?
+      filepath
+    end
+
+    ##
+    # When submitting location data (a.k.a. the "based near" attribute) via the UI,
+    # Hyrax expects to receive a `based_near_attributes` hash in a specific format.
+    # We need to take geonames urls as provided by the customer and transform them to
+    # mimic what the Hyrax UI would ordinarily produce. These will get turned into
+    # Hyrax::ControlledVocabularies::Location objects upon ingest.
+    # The expected hash looks like this:
+    #   "based_near_attributes"=>
+    #     {
+    #       "0"=> {
+    #               "id"=>"http://sws.geonames.org/5667009/", "_destroy"=>""
+    #             },
+    #       "1"=> {
+    #               "id"=>"http://sws.geonames.org/6252001/", "_destroy"=>""
+    #             },
+    #   }
+    # @return [Hash] a "based_near_attributes" hash as
+    def based_near_attributes(based_near)
+      original_geonames_uris = based_near
+      return if original_geonames_uris.empty?
+      based_near_attributes = {}
+      original_geonames_uris.each_with_index do |uri, i|
+        based_near_attributes[i.to_s] = { 'id' => uri_to_sws(uri), "_destroy" => "" }
+      end
+      based_near_attributes
+    end
+
+    #
+    # Take a user-facing geonames URI and return an sws URI, of the form Hyrax expects
+    # (e.g., "http://sws.geonames.org/6252001/")
+    # @param [String] uri
+    # @return [String] an sws style geonames uri
+    def uri_to_sws(uri)
+      uri = URI(uri)
+      geonames_number = uri.path.split('/')[1]
+      "http://sws.geonames.org/#{geonames_number}/"
+    end
+
+    private
+
+      # Build a pared down actor stack that will not re-attach files,
+      # or set workflow, or do anything except update metadata.
+      # TODO: We should be able to set an environment variable that would allow updates to go through the regular
+      # actor stack instead of the stripped down one, in case we want to re-import files.
+      def metadata_only_middleware
+        terminator = Hyrax::Actors::Terminator.new
+        Zizia::MetadataOnlyStack.build_stack.build(terminator)
+      end
+
+      # Update an existing object using the Hyrax actor stack
+      # We assume the object was created as expected if the actor stack returns true.
+      # Note that for now the update stack will only update metadata and update collection membership, it will not re-import files.
+      def update_for(existing_record:, update_record:)
+        info_stream << "event: record_update_started, batch_id: #{batch_id}, collection_id: #{collection_id}, #{deduplication_field}: #{update_record.respond_to?(deduplication_field) ? update_record.send(deduplication_field) : update_record}"
+        additional_attrs = {
+          depositor: @depositor.user_key
+        }
+        attrs = update_record.attributes.merge(additional_attrs)
+        attrs = attrs.merge(member_of_collections_attributes: { '0' => { id: collection_id } }) if collection_id
+        # Ensure nothing is passed in the files field,
+        # since this is reserved for Hyrax and is where uploaded_files will be attached
+        attrs.delete(:files)
+
+        # We aren't using the attach remote files actor, so make sure any remote files are removed from the params before we try to save the object.
+        attrs.delete(:remote_files)
+
+        based_near = attrs.delete(:based_near)
+        attrs = attrs.merge(based_near_attributes: based_near_attributes(based_near)) unless based_near.nil? || based_near.empty?
+
+        actor_env = Hyrax::Actors::Environment.new(existing_record, ::Ability.new(@depositor), attrs)
+        if metadata_only_middleware.update(actor_env)
+          info_stream << "event: record_updated, batch_id: #{batch_id}, record_id: #{existing_record.id}, collection_id: #{collection_id}, #{deduplication_field}: #{existing_record.respond_to?(deduplication_field) ? existing_record.send(deduplication_field) : existing_record}"
+          @success_count += 1
+        else
+          existing_record.errors.each do |attr, msg|
+            error_stream << "event: validation_failed, batch_id: #{batch_id}, collection_id: #{collection_id}, attribute: #{attr.capitalize}, message: #{msg}, record_title: record_title: #{attrs[:title] ? attrs[:title] : attrs}"
+          end
+          @failure_count += 1
+        end
+      end
+
+      # Create an object using the Hyrax actor stack
+      # We assume the object was created as expected if the actor stack returns true.
+      def create_for(record:)
+        info_stream << "event: record_import_started, batch_id: #{batch_id}, collection_id: #{collection_id}, record_title: #{record.respond_to?(:title) ? record.title : record}"
+
+        additional_attrs = {
+          uploaded_files: create_upload_files(record),
+          depositor: @depositor.user_key
+        }
+
+        created = import_type.new
+
+        attrs = record.attributes.merge(additional_attrs)
+        attrs = attrs.merge(member_of_collections_attributes: { '0' => { id: collection_id } }) if collection_id
+
+        # Ensure nothing is passed in the files field,
+        # since this is reserved for Hyrax and is where uploaded_files will be attached
+        attrs.delete(:files)
+
+        based_near = attrs.delete(:based_near)
+        attrs = attrs.merge(based_near_attributes: based_near_attributes(based_near)) unless based_near.nil? || based_near.empty?
+
+        actor_env = Hyrax::Actors::Environment.new(created,
+                                                   ::Ability.new(@depositor),
+                                                   attrs)
+
+        if Hyrax::CurationConcern.actor.create(actor_env)
+          info_stream << "event: record_created, batch_id: #{batch_id}, record_id: #{created.id}, collection_id: #{collection_id}, record_title: #{attrs[:title]&.first}"
+          @success_count += 1
+        else
+          created.errors.each do |attr, msg|
+            error_stream << "event: validation_failed, batch_id: #{batch_id}, collection_id: #{collection_id}, attribute: #{attr.capitalize}, message: #{msg}, record_title: record_title: #{attrs[:title] ? attrs[:title] : attrs}"
+          end
+          @failure_count += 1
+        end
+      end
+  end
+end

data/lib/zizia/importer.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Zizia
+  ##
+  # The chief entry point for bulk import of records. `Importer` accepts a
+  # {Parser} on initialization and iterates through its {Parser#records}, importing
+  # each using a given {RecordImporter}.
+  #
+  # @example Importing in bulk from a CSV file
+  #   parser = Zizia::Parser.for(file: File.new('path/to/import.csv'))
+  #
+  #   Zizia::Importer.new(parser: parser).import if parser.validate
+  #
+  class Importer
+    extend Forwardable
+
+    ##
+    # @!attribute [rw] parser
+    #   @return [Parser]
+    # @!attribute [rw] record_importer
+    #   @return [RecordImporter]
+    attr_accessor :parser, :record_importer
+
+    ##
+    # @!method records()
+    #   @see Parser#records
+    def_delegator :parser, :records, :records
+
+    ##
+    # @param parser          [Parser] The parser to use as the source for import
+    #   records.
+    # @param record_importer [RecordImporter] An object to handle import of
+    #   each record
+    def initialize(parser:, record_importer: RecordImporter.new, info_stream: Zizia.config.default_info_stream, error_stream: Zizia.config.default_error_stream)
+      self.parser          = parser
+      self.record_importer = record_importer
+      @info_stream = info_stream
+      @error_stream = error_stream
+    end
+
+    # Do not attempt to run an import if there are no records. Instead, just write to the log.
+    def no_records_message
+      @info_stream << "event: empty_import, batch_id: #{record_importer.batch_id}"
+      @error_stream << "event: empty_import, batch_id: #{record_importer.batch_id}"
+    end
+
+    ##
+    # Import each record in {#records}.
+    #
+    # @return [void]
+    def import
+      no_records_message && return unless records.count.positive?
+      start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      @info_stream << "event: start_import, batch_id: #{record_importer.batch_id}, expecting to import #{records.count} records."
+      records.each { |record| record_importer.import(record: record) }
+      end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      elapsed_time = end_time - start_time
+      @info_stream << "event: finish_import, batch_id: #{record_importer.batch_id}, successful_record_count: #{record_importer.success_count}, failed_record_count: #{record_importer.failure_count}, elapsed_time: #{elapsed_time}, elapsed_time_per_record: #{elapsed_time / records.count}"
+    end
+  end
+end

data/lib/zizia/input_record.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Zizia
+  ##
+  # @example Building an importer with the factory
+  #   record = InputRecord.from({some: :metadata}, mapper: MyMapper.new)
+  #   record.some # => :metadata
+  #
+  class InputRecord
+    ##
+    # @!attribute [rw] mapper
+    #   @return [#map_fields]
+    attr_accessor :mapper
+
+    ##
+    # @param mapper [#map_fields]
+    def initialize(mapper: HyraxBasicMetadataMapper.new)
+      self.mapper = mapper
+    end
+
+    class << self
+      ##
+      # @param metadata [Object]
+      # @param mapper  [#map_fields]
+      #
+      # @return [InputRecord] an input record mapping metadata with the given
+      #   mapper
+      def from(metadata:, mapper: HyraxBasicMetadataMapper.new)
+        mapper.metadata = metadata
+        new(mapper: mapper)
+      end
+    end
+
+    ##
+    # @return [Hash<Symbol, Object>]
+    def attributes
+      mapper.fields.each_with_object({}) do |field, attrs|
+        attrs[field] = public_send(field)
+      end
+    end
+
+    ##
+    # @return [String, nil] an identifier for the representative file; nil if
+    #   none is given.
+    def representative_file
+      return mapper.representative_file if
+        mapper.respond_to?(:representative_file)
+
+      nil
+    end
+
+    ##
+    # Respond to methods matching mapper fields
+    def method_missing(method_name, *args, &block)
+      return super unless mapper.field?(method_name)
+      mapper.public_send(method_name)
+    end
+
+    ##
+    # @see #method_missing
+    def respond_to_missing?(method_name, include_private = false)
+      mapper.field?(method_name) || super
+    end
+  end
+end

data/lib/zizia/log_stream.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Zizia
+  class LogStream
+    ##
+    # @!attribute [rw] logger
+    #   @return [Logger]
+    # @!attribute [rw] severity
+    #   @return [Logger::Serverity]
+    attr_accessor :logger, :severity
+
+    def initialize(logger: nil, severity: nil)
+      self.logger   = logger   || Logger.new(build_filename)
+      self.severity = severity || Logger::INFO
+    end
+
+    def <<(msg)
+      logger.add(severity, msg)
+      STDOUT << msg
+    end
+
+    private
+
+      def build_filename
+        return ENV['IMPORT_LOG'] if ENV['IMPORT_LOG']
+        return rails_log_name if rails_log_name
+        './log/zizia_import.log'
+      end
+
+      def rails_log_name
+        case Rails.env
+        when 'production'
+          Rails.root.join('log', "csv_import.log").to_s
+        when 'development'
+          Rails.root.join('log', "dev_csv_import.log").to_s
+        when 'test'
+          Rails.root.join('log', "test_csv_import.log").to_s
+        end
+      rescue ::NameError
+        false
+      end
+  end
+end

data/lib/zizia/metadata_mapper.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Zizia
+  ##
+  # A null/base mapper that maps no fields.
+  #
+  # Real mapper implementations need to provide `#fields`, enumerating over
+  # `Symbols` that represent the fields on the target object (e.g. an
+  # `ActiveFedora::Base`/`Hyrax::WorkBehavior`) that the mapper can handle.
+  # For each field in `#fields`, the mapper must respond to a matching method
+  # (i.e. `:title` => `#title`), and return the value(s) that should be set to
+  # the target's attributes upon mapping.
+  #
+  # To ease the implementation of field methods, this base class provides
+  # a `#method_missing` that forwards missing method names to a `#map_field`
+  # method. `#map_field` can be implemented to provide a generalized field
+  # mapping when a common pattern will be used for many methods. Callers should
+  # avoid relying on this protected method directly, since mappers may implement
+  # individual field methods in any other way (e.g. `def title; end`) to route
+  # around `#map_field`. Implementations are also free to override
+  # `#method_missing` if desired.
+  #
+  # Mappers generally operate over some input `#metadata`. Example metadata
+  # types that mappers could be implemented over include `Hash`, `CSV`, `XML`,
+  # `RDF::Graph`, etc...; mappers are free to interpret or ignore the contents
+  # of their underlying metadata data structures at their leisure. Values for
+  # fields are /usually/ derived from the `#metadata`, but can also be generated
+  # from complex logic or even hard-coded.
+  #
+  # @example Using a MetadataMapper
+  #   mapper = MyMapper.new
+  #   mapper.metadata = some_metadata_object
+  #   mapper.fields # => [:title, :author, :description]
+  #
+  #   mapper.title # => 'Some Title'
+  #
+  #   mapper.fields.map { |field| mapper.send(field) }
+  #
+  # @see ImportRecord#attributes for the canonical usage of a `MetadataMapper`.
+  # @see HashMapper for an example implementation with dynamically generated
+  #   fields
+  class MetadataMapper
+    ##
+    # @!attribute [rw] metadata
+    #   @return [Object]
+    attr_accessor :metadata
+
+    ##
+    # @param name [Symbol]
+    #
+    # @return [Boolean]
+    def field?(name)
+      fields.include?(name)
+    end
+
+    ##
+    # @return [Enumerable<Symbol>] The fields the mapper can process
+    def fields
+      []
+    end
+
+    def method_missing(method_name, *args, &block)
+      return map_field(method_name) if fields.include?(method_name)
+      super
+    end
+
+    def respond_to_missing?(method_name, include_private = false)
+      field?(method_name) || super
+    end
+
+    protected
+
+      ##
+      # @private
+      #
+      # @param name [Symbol]
+      #
+      # @return [Object]
+      def map_field(_name)
+        raise NotImplementedError
+      end
+  end
+end

data/lib/zizia/metadata_only_stack.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Zizia
+  class MetadataOnlyStack
+    def self.build_stack
+      ActionDispatch::MiddlewareStack.new.tap do |middleware|
+        # Wrap everything in a database transaction, if the save of the resource
+        # fails then roll back any database AdminSetChangeSet
+        middleware.use Hyrax::Actors::TransactionalRequest
+
+        # Ensure you are mutating the most recent version
+        # middleware.use Hyrax::Actors::OptimisticLockValidator
+
+        # Attach files from a URI (for BrowseEverything)
+        # middleware.use Hyrax::Actors::CreateWithRemoteFilesActor
+
+        # Attach files uploaded in the form to the UploadsController
+        # In Californica, for command line import,
+        # we are using the CreateWithFilesActor to attach
+        # local files with a file:// url, not via the UploadsController,
+        # so we never use the CreateWithFilesActor
+        # middleware.use Hyrax::Actors::CreateWithFilesActor
+
+        # Add/remove the resource to/from a collection
+        middleware.use Hyrax::Actors::CollectionsMembershipActor
+
+        # Add/remove to parent work
+        # middleware.use Hyrax::Actors::AddToWorkActor
+
+        # Add/remove children (works or file_sets)
+        # middleware.use Hyrax::Actors::AttachMembersActor
+
+        # Set the order of the children (works or file_sets)
+        # middleware.use Hyrax::Actors::ApplyOrderActor
+
+        # Sets the default admin set if they didn't supply one
+        # middleware.use Hyrax::Actors::DefaultAdminSetActor
+
+        # Decode the private/public/institution on the form into permisisons on
+        # the model
+        # We aren't using this in Californica, we're just setting the visibility
+        # at import time
+        # middleware.use Hyrax::Actors::InterpretVisibilityActor
+
+        # Handles transfering ownership of works from one user to another
+        # We aren't using this in Californica
+        # middleware.use Hyrax::Actors::TransferRequestActor
+
+        # Copies default permissions from the PermissionTemplate to the work
+        # middleware.use Hyrax::Actors::ApplyPermissionTemplateActor
+
+        # Remove attached FileSets when destroying a work
+        # middleware.use Hyrax::Actors::CleanupFileSetsActor
+
+        # Destroys the trophies in the database when the work is destroyed
+        # middleware.use Hyrax::Actors::CleanupTrophiesActor
+
+        # Destroys the feature tag in the database when the work is destroyed
+        # middleware.use Hyrax::Actors::FeaturedWorkActor
+
+        # Persist the metadata changes on the resource
+        middleware.use Hyrax::Actors::ModelActor
+
+        # Start the workflow for this work
+        # middleware.use Hyrax::Actors::InitializeWorkflowActor
+      end
+    end
+    # rubocop:enable Metrics/MethodLength
+  end
+end