rof 1.0.7 → 1.2.0

data/lib/rof/translators/jsonld_to_rof/predicate_handler.rb

@@ -0,0 +1,223 @@
+require 'active_support/core_ext/array/wrap'
+
+module ROF
+  module Translators
+    module JsonldToRof
+      # Responsible for dealing with registered predicates and how those are handled.
+      module PredicateHandler
+        class UnhandledPredicateError < RuntimeError
+          def initialize(predicate, urls)
+            super(%(Unable to handle predicate "#{predicate}". The following predicate URLs were registered #{urls.inspect}))
+          end
+        end
+
+        # @api public
+        #
+        # Parse the RDF predicate and RDF object and add it's contents to the accumulator
+        #
+        # @example
+        #   Given the following 4 RDF N-Triples (subject, predicate, object). The first two, with subject "_:b0" represent blank nodes.
+        #   The last one with subject "<https://curate.nd.edu/show/zk51vd69n1r>" has an object that points to the "_:b0" blank node.
+        #     _:b0 <http://purl.org/dc/terms/contributor> "David R.Hyde" .
+        #     _:b0 <http://www.ndltd.org/standards/metadata/etdms/1.1/role> "Research Director" .
+        #     <https://curate.nd.edu/show/zk51vd69n1r> <http://purl.org/dc/terms/contributor> _:b0 .
+        #     <https://curate.nd.edu/show/zk51vd69n1r> <http://projecthydra.org/ns/relations#hasEditorGroup> <https://curate.nd.edu/show/q524jm23g92> .
+        #   For the first two N-Triples you would get a BlankNodeHandler; For the last two, you would get a UriSubjectHandler
+        #
+        # @note It is assumed that all blank nodes (e.g. RDF::Node) will be processed before you process any RDF::URI nodes.
+        #
+        # @param [RDF::Predicate] predicate - the RDF predicate that we will parse and add to the appropriate spot in the accumulator
+        # @param [RDF::Object] object - the RDF object that we will parse and add to the appropriate spot in the accumulator
+        # @param [ROF::Translators::JsonldToRof::Accumulator] accumulator - a data accumulator that will be changed in place
+        # @return [ROF::Translators::JsonldToRof::Accumulator] the given accumulator
+        # @raise [ROF::Translators::JsonldToRof::UnhandledPredicateError] when we are unable to handle the given predicate
+        def self.call(predicate, object, accumulator, blank_node = false)
+          handler = registry.handler_for(predicate)
+          handler.handle(object, accumulator, blank_node)
+          accumulator
+        end
+
+        # @api public
+        # @param [String] url - The URL that we want to match against
+        # @yield The block to configure how we handle RDF Predicates that match the gvien URL
+        # @yieldparam [ROF::JsonldToRof::PredicateHandler::UrlHandler]
+        # @see ./spec/lib/rof/translators/jsonld_to_rof/predicate_handler_spec.rb for details and usage usage
+        def self.register(url, &block)
+          registry << UrlHandler.new(url, &block)
+        end
+
+        # @api private
+        def self.registry
+          @registry ||= RegistrySet.new
+        end
+        private_class_method :registry
+
+        def self.clear_registry!(set_with = RegistrySet.new)
+          @registry = set_with
+        end
+        private_class_method :clear_registry!
+
+        class RegistrySet
+          def initialize
+            @set = []
+          end
+
+          def <<(value)
+            @set << value
+          end
+
+          def handler_for(predicate)
+            location_extractor = nil
+            @set.each do |handler|
+              location_extractor = handler.location_extractor_for(predicate)
+              break if location_extractor
+            end
+            raise UnhandledPredicateError.new(predicate, @set.map(&:url)) if location_extractor.nil?
+            location_extractor
+          end
+        end
+        private_constant :RegistrySet
+
+        # For a given URL map all of the predicates; Some predicates require explicit mapping, while others
+        # may use implicit mapping.
+        class UrlHandler
+          def initialize(url)
+            @url = url
+            @within = []
+            @namespace_prefix = ''
+            @slug_handlers = {}
+            yield(self) if block_given?
+          end
+          attr_reader :url
+
+          # The final key in the location array should be prefixed with the namespace_prefix; By default this is ""
+          # @param [String, nil] prefix - what is the namespace prefix to apply to the last location in the array.
+          # @return [String]
+          def namespace_prefix(prefix = nil)
+            return @namespace_prefix if prefix.nil?
+            @namespace_prefix = prefix
+          end
+
+          # Prepend the within array to the location array
+          # @param [Array<String>, nil] location - where in the ROF document are we putting the value
+          # @return [Array<String>]
+          def within(location = nil)
+            return @within if location.nil?
+            @within = Array.wrap(location)
+          end
+
+          # @param [#to_s] predicate
+          # @return [nil, LocationExtractor] if the given predicate does not match the url, return nil; Otherwise return a LocationExtractor
+          # @see LocationExtractor
+          def location_extractor_for(predicate)
+            return nil unless predicate.to_s =~ %r{^#{Regexp.escape(@url)}(.*)}
+            slug = $1
+            handlers = handlers_for(slug)
+            LocationExtractor.new(predicate, handlers)
+          end
+
+          private
+
+          # @param [String] slug - a slug that may or may not have been registered
+          # @return [Array<#call>] an array of handlers that each respond to #call
+          # @see ImplicitLocationHandler
+          # @see ExplicitLocationSlugHandler
+          # @see BlockSlugHandler
+          def handlers_for(slug)
+            Array.wrap(@slug_handlers.fetch(slug) { ImplicitLocationHandler.new(self, slug) })
+          end
+
+          public
+
+          # @param [String] slug =
+          # @param [Hash] options (with symbol keys)
+          # @option options [Boolean] :force - don't apply the within nor namespace prefix
+          # @option options [Array] :to - an array that will be nested Hash keys
+          # @yield If a block is given, call the block (and skip all other configuration)
+          # @yieldparam [String] object
+          # @see BlockSlugHandler for details concerning a mapping via a block
+          # @see ExplicitLocationSlugHandler for details concerning a mapping via a to: option
+          def map(slug, options = {}, &block)
+            @slug_handlers ||= {}
+            @slug_handlers[slug] ||= []
+            if block_given?
+              @slug_handlers[slug] << BlockSlugHandler.new(self, slug, options, block)
+            else
+              @slug_handlers[slug] << ExplicitLocationSlugHandler.new(self, slug, options)
+            end
+          end
+
+          # Responsible for coordinating the extraction of the
+          class LocationExtractor
+            def initialize(predicate, handlers)
+              @predicate = predicate
+              @handlers = Array.wrap(handlers)
+            end
+
+            def handle(object, accumulator, blank_node)
+              @handlers.each do |handler|
+                handler.call(object, accumulator, blank_node)
+              end
+              accumulator
+            end
+          end
+          private_constant :LocationExtractor
+
+          class ImplicitLocationHandler
+            def initialize(url_handler, slug)
+              @url_handler = url_handler
+              @slug = slug
+            end
+            attr_reader :slug
+            def call(object, accumulator, blank_node)
+              to = @url_handler.within + Array.wrap(slug)
+              to[-1] = "#{@url_handler.namespace_prefix}#{to[-1]}"
+              accumulator.add_predicate_location_and_value(to, object, blank_node)
+            end
+          end
+          private_constant :ImplicitLocationHandler
+
+          class BlockSlugHandler
+            def initialize(url_handler, slug, options, block)
+              @url_handler = url_handler
+              @slug = slug
+              @options = options
+              @block = block
+            end
+            attr_reader :slug
+
+            # @todo Are there differences that need to be handled for the blank_node?
+            def call(object, accumulator, _blank_node)
+              @block.call(object, accumulator)
+            end
+          end
+          private_constant :BlockSlugHandler
+
+          class ExplicitLocationSlugHandler
+            def initialize(url_handler, slug, options)
+              @url_handler = url_handler
+              @slug = slug
+              @options = options
+            end
+            attr_reader :slug
+
+            def call(object, accumulator, blank_node)
+              to = @options.fetch(:to)
+              unless force?
+                to = @url_handler.within + Array.wrap(to)
+                to[-1] = "#{@url_handler.namespace_prefix}#{to[-1]}"
+              end
+              accumulator.add_predicate_location_and_value(to, object, blank_node)
+            end
+
+            def force?
+              @options.fetch(:force, false)
+            end
+          end
+          private_constant :ExplicitLocationSlugHandler
+        end
+        private_constant :UrlHandler
+      end
+    end
+  end
+end

data/lib/rof/translators/jsonld_to_rof/predicate_object_handler.rb

@@ -0,0 +1,125 @@
+require 'rdf'
+require 'rof/translators/jsonld_to_rof/predicate_handler'
+
+module ROF
+  module Translators
+    module JsonldToRof
+      # We need to handle the Predicate / Object pair as one (thank you RDF blank nodes for this nuance)
+      module PredicateObjectHandler
+        # @api public
+        #
+        # Parse the RDF::Predicate, RDF::Object and the relevant data to the contents to the accumulator
+        #
+        # @example
+        #   Given the following 4 RDF N-Triples (subject, predicate, object). The first and second RDF objects are RDF::Literal. The 3rd triple's object is
+        #   and RDF::Node. And the last is an RDF::URI. Each require different handlers as they have nuanced differences.
+        #     _:b0 <http://purl.org/dc/terms/contributor> "David R.Hyde" .
+        #     _:b0 <http://www.ndltd.org/standards/metadata/etdms/1.1/role> "Research Director" .
+        #     <https://curate.nd.edu/show/zk51vd69n1r> <http://purl.org/dc/terms/contributor> _:b0 .
+        #     <https://curate.nd.edu/show/zk51vd69n1r> <http://projecthydra.org/ns/relations#hasEditorGroup> <https://curate.nd.edu/show/q524jm23g92> .
+        #
+        # @note It is assumed that all blank nodes (e.g. RDF::Node) will be processed before you process any RDF::URI nodes.
+        #
+        # @param [RDF::Predicate] predicate - the RDF predicate that we will parse and add to the appropriate spot in the accumulator
+        # @param [RDF::Object] object - the RDF object that we will parse and add to the appropriate spot in the accumulator
+        # @param [ROF::Translators::JsonldToRof::Accumulator] accumulator - a data accumulator that will be changed in place
+        # @return [ROF::Translators::JsonldToRof::Accumulator] the given accumulator
+        # @raise [ROF::Translators::JsonldToRof::UnknownRdfObjectTypeError] when the RDF::Object's subject is not a valid type
+        def self.call(predicate, object, accumulator, options = {})
+          new(predicate, object, accumulator, options).call
+          accumulator
+        end
+
+        # @api private
+        #
+        # @param [RDF::Predicate] predicate - the RDF predicate that we will parse and add to the appropriate spot in the accumulator
+        # @param [RDF::Object] object - the RDF object that we will parse and add to the appropriate spot in the accumulator
+        # @param [ROF::Translators::JsonldToRof::Accumulator] accumulator - a data accumulator that will be changed in place
+        # @return [#call]
+        def self.new(predicate, object, accumulator, options)
+          klass_for(object).new(predicate, object, accumulator, options)
+        end
+
+        class UnknownRdfObjectTypeError < RuntimeError
+        end
+
+        # @api private
+        def self.klass_for(object)
+          case object
+          when RDF::URI
+            UriPredicateObjectHandler
+          when RDF::Node
+            NodePredicateObjectHandler
+          when RDF::Literal
+            LiteralPredicateObjectHandler
+          else
+            raise UnknownRdfObjectTypeError, "Unable to determine object handler for #{object.inspect}"
+          end
+        end
+
+        # @api private
+        class UriPredicateObjectHandler
+          def initialize(predicate, object, accumulator, options)
+            @predicate = predicate
+            @object = object
+            @accumulator = accumulator
+            @options = options
+          end
+
+          def call
+            PredicateHandler.call(predicate, object, accumulator, options[:blank_node])
+            accumulator
+          end
+
+          private
+          attr_reader :predicate, :object, :accumulator, :options
+        end
+        private_constant :UriPredicateObjectHandler
+
+        # @api private
+        # Blank Nodes; Oh how we love thee. Let me count the ways
+        class NodePredicateObjectHandler
+          def initialize(predicate, object, accumulator, options)
+            @predicate = predicate
+            @object = object
+            @accumulator = accumulator
+            @options = options
+          end
+
+          def call
+            blank_node = accumulator.fetch_blank_node(object)
+            blank_node.each_pair do |blank_node_predicate, blank_node_objects|
+              blank_node_objects.each do |blank_node_object|
+                PredicateObjectHandler.call(blank_node_predicate, blank_node_object, accumulator, blank_node: object)
+              end
+            end
+            accumulator
+          end
+
+          private
+          attr_reader :predicate, :object, :accumulator, :options
+        end
+        private_constant :NodePredicateObjectHandler
+
+        # @api private
+        class LiteralPredicateObjectHandler
+          def initialize(predicate, object, accumulator, options)
+            @predicate = predicate
+            @object = object
+            @accumulator = accumulator
+            @options = options
+          end
+
+          def call
+            PredicateHandler.call(predicate, object, accumulator, options[:blank_node])
+            accumulator
+          end
+
+          private
+          attr_reader :predicate, :object, :accumulator, :options
+        end
+        private_constant :LiteralPredicateObjectHandler
+      end
+    end
+  end
+end

data/lib/rof/translators/jsonld_to_rof/statement_handler.rb

@@ -0,0 +1,91 @@
+require 'rof/translators/jsonld_to_rof/predicate_object_handler'
+
+module ROF
+  module Translators
+    module JsonldToRof
+      # Responsible for parsing an RDF statement and adding to the accumulator.
+      module StatementHandler
+        # @api public
+        #
+        # Parse the RDF statement and add it's contents to the accumulator
+        #
+        # @example
+        #   Given the following 4 RDF N-Triples (subject, predicate, object). The first two, with subject "_:b0" represent blank nodes.
+        #   The last one with subject "<https://curate.nd.edu/show/zk51vd69n1r>" has an object that points to the "_:b0" blank node.
+        #     _:b0 <http://purl.org/dc/terms/contributor> "David R.Hyde" .
+        #     _:b0 <http://www.ndltd.org/standards/metadata/etdms/1.1/role> "Research Director" .
+        #     <https://curate.nd.edu/show/zk51vd69n1r> <http://purl.org/dc/terms/contributor> _:b0 .
+        #     <https://curate.nd.edu/show/zk51vd69n1r> <http://projecthydra.org/ns/relations#hasEditorGroup> <https://curate.nd.edu/show/q524jm23g92> .
+        #   For the first two N-Triples you would get a BlankNodeHandler; For the last two, you would get a UriSubjectHandler
+        #
+        # @note It is assumed that all blank nodes (e.g. RDF::Node) will be processed before you process any RDF::URI nodes.
+        #
+        # @param [RDF::Statement] statement - the RDF statement that we will parse and add to the appropriate spot in the accumulator
+        # @param [ROF::Translators::JsonldToRof::Accumulator] accumulator - a data accumulator that will be changed in place
+        # @return [ROF::Translators::JsonldToRof::Accumulator] the given accumulator
+        # @raise [ROF::Translators::JsonldToRof::UnhandledRdfSubjectError] when the RDF::Statement's subject is not a valid type
+        def self.call(statement, accumulator)
+          new(statement, accumulator).call
+          accumulator
+        end
+
+        class UnhandledRdfSubjectError < RuntimeError
+        end
+
+        # @api private
+        def self.new(statement, accumulator)
+          case statement.subject
+          when RDF::URI
+            UriSubjectHandler.new(statement, accumulator)
+          when RDF::Node
+            BlankNodeHandler.new(statement, accumulator)
+          else
+            raise UnhandledRdfSubjectError, "Unable to determine subject handler for #{statement.inspect}"
+          end
+        end
+
+        # Responsible for accumulating the ROF data for a URI based resource
+        class UriSubjectHandler
+          def initialize(statement, accumulator)
+            @accumulator = accumulator
+            @statement = statement
+          end
+
+          def call
+            handle_subject
+            handle_predicate_and_object
+            @accumulator
+          end
+
+          private
+
+          def handle_predicate_and_object
+            PredicateObjectHandler.call(@statement.predicate, @statement.object, @accumulator)
+          end
+
+          def handle_subject
+            return nil unless @statement.subject.to_s =~ %r{https://curate.nd.edu/show/([^\\]+)/?}
+            pid = "und:#{$1}"
+            @accumulator.add_pid(pid)
+          end
+        end
+        private_constant :UriSubjectHandler
+
+        # Responsible for handling blank nodes in the RDF graph; Examples include ETD degree information
+        # Blank node subjects behave different from UriSubjectHandler
+        class BlankNodeHandler
+          def initialize(statement, accumulator)
+            @accumulator = accumulator
+            @statement = statement
+          end
+
+          def call
+            @accumulator.add_blank_node(@statement)
+            @accumulator
+          end
+        end
+        private_constant :BlankNodeHandler
+      end
+    end
+  end
+end

data/lib/rof/translators/osf_to_rof.rb

@@ -0,0 +1,191 @@
+require 'json'
+require 'zlib'
+require 'rubygems/package'
+require 'rdf/turtle'
+require 'rof/osf_context'
+require 'rof/rdf_context'
+require 'rof/utility'
+require 'rof/translator'
+
+module ROF::Translators
+  # Class for managing OSF Archive data transformations
+  # It is called after the get-from-osf task, and before the work-xlat task
+  class OsfToRof < ROF::Translator
+    # @todo Set this to be something more meaningful than an empty lambda
+    # @return [#call]
+    def self.default_previously_archived_pid_finder
+      ->(archive_type, osf_project_identifier) {}
+    end
+
+    # Convert Osf Archive tar.gz  to ROF
+    def self.call(project, config, previously_archived_pid_finder = default_previously_archived_pid_finder)
+      new(project, config, previously_archived_pid_finder).call
+    end
+
+    def initialize(project, config, previously_archived_pid_finder = self.class.default_previously_archived_pid_finder)
+      @config = config
+      @project = project
+      @previously_archived_pid_finder = previously_archived_pid_finder
+      @osf_map = ROF::OsfToNDMap
+    end
+
+    def call
+      rof_array = []
+      return {} if project.nil?
+      @ttl_data = ttl_from_targz(source_slug + '.ttl')
+      rof_array[0] = build_archive_record
+      rof_array
+    end
+
+    # @api private
+    # @see https://github.com/ndlib/curate_nd/blob/677c05c836ff913c01dcbbfc5e5d21366b87d500/app/repository_models/osf_archive.rb#L62
+    def archive_type
+      project.fetch('package_type')
+    end
+
+    # @api private
+    # This is a bit of a misnomer; As used it represents the path to the project or registration
+    # that we have ingested (e.g. https://osf.io/:source_slug)
+    #
+    # It was previously named :project_identifier in this class, but that gets conflated with the
+    # underlying object's osf_project_identifier (e.g. what OSF Project was this archive originally based on)
+    #
+    # @see https://github.com/ndlib/curate_nd/blob/115efec2e046257282a86fe2cd98c7d229d04cf9/app/repository_models/osf_archive.rb#L96
+    def source_slug
+      project.fetch('project_identifier')
+    end
+
+    # @api private
+    # @see https://github.com/ndlib/curate_nd/blob/115efec2e046257282a86fe2cd98c7d229d04cf9/app/repository_models/osf_archive.rb#L106
+    def osf_project_identifier
+      return source_slug if project['package_type'] == 'OSF Project'
+      osf_url_from_filename(ttl_data[0][@osf_map['registeredFrom']][0]['@id'])
+    end
+
+    private
+
+    attr_reader :config, :project
+
+    # A function responsible for finding the previously archive pid.
+    # @return [#call]
+    # @see #default_previously_archived_pid_finder for interface
+    attr_reader :previously_archived_pid_finder
+
+    # this is an array- the addition elements are the contributor(s)
+    # @return [Array<Hash>]
+    # @see #ttl_from_targz
+    attr_reader :ttl_data
+
+    # reads a ttl file and makes it a JSON-LD file that we can parse
+    def fetch_from_ttl(ttl_file)
+      graph = RDF::Turtle::Reader.open(ttl_file, prefixes:  ROF::OsfPrefixList.dup)
+      JSON::LD::API.fromRdf(graph)
+    end
+
+    # extracts given ttl file from JHU tar.gz package
+    # - assumed to live under data/obj/root
+    # @return [Array<Hash>] the first element is the "work" and the additional elements, if any, are the contributor(s)
+    def ttl_from_targz(ttl_filename)
+      package_dir = config.fetch('package_dir')
+      ttl_path = File.join(source_slug, 'data/obj/root', ttl_filename)
+      ROF::Utility.file_from_targz(File.join(package_dir, source_slug + '.tar.gz'), ttl_path)
+      fetch_from_ttl(File.join(package_dir, ttl_path))
+    end
+
+    # Maps RELS-EXT
+    def map_rels_ext
+      rels_ext = {}
+      rels_ext['@context'] = ROF::RelsExtRefContext.dup
+      apply_previous_archived_version_if_applicable(rels_ext)
+      rels_ext
+    end
+
+    # sets metadata
+    def map_metadata
+      metadata = {}
+      metadata['@context'] = ROF::RdfContext.dup
+      # metdata derived from project ttl file
+      metadata['dc:created'] = Time.iso8601(ttl_data[0][@osf_map['dc:created']][0]['@value']).to_date.iso8601 + 'Z'
+      metadata['dc:title'] = ttl_data[0][@osf_map['dc:title']][0]['@value']
+      metadata['dc:description'] = ttl_data[0][@osf_map['dc:description']][0]['@value']
+      metadata['dc:subject'] = map_subject
+      # metadata derived from osf_projects data, passed from UI
+      metadata['dc:source'] = 'https://osf.io/' + source_slug
+      metadata['dc:creator#adminstrative_unit'] = project['administrative_unit']
+      metadata['dc:creator#affiliation'] = project['affiliation']
+      metadata['nd:osfProjectIdentifier'] = osf_project_identifier
+      metadata['dc:creator'] = map_creator
+      metadata['dc:type'] = project['package_type']
+      metadata
+    end
+
+    # For reference to the assumed RELS-EXT see the following spec in CurateND
+    # @see https://github.com/ndlib/curate_nd/blob/115efec2e046257282a86fe2cd98c7d229d04cf9/spec/repository_models/osf_archive_spec.rb#L97
+    def apply_previous_archived_version_if_applicable(rels_ext)
+      # If a previously archived pid was passed in, use it to set pav:previousVersion
+      # If not, check SOLR for one.
+      pid = previously_archived_pid_finder.call(archive_type, osf_project_identifier)
+      pid = ROF::Utility.check_solr_for_previous(config, osf_project_identifier) if pid.nil?
+      rels_ext['pav:previousVersion'] = pid if pid
+      rels_ext
+    end
+
+    # Constructs OsfArchive Record from ttl_data, data from the UI form,
+    # and task config data
+    def build_archive_record
+      this_rof = {}
+      this_rof['owner'] = project['owner']
+      this_rof['type'] = 'OsfArchive'
+      this_rof['rights'] = map_rights
+      this_rof['rels-ext'] = map_rels_ext
+      this_rof['metadata'] = map_metadata
+      this_rof['files'] = [source_slug + '.tar.gz']
+      this_rof
+    end
+
+    # sets subject
+    def map_subject
+      if ttl_data[0].key?(@osf_map['dc:subject'])
+        return ttl_data[0][@osf_map['dc:subject']][0]['@value']
+      end
+      ''
+    end
+
+    # make osf url from bagfile name
+    def osf_url_from_filename(ttl_file)
+      project_id = ttl_file.rpartition('/')[2].rpartition('.')[0]
+      project_id
+    end
+
+    # figures out the rights
+    def map_rights
+      rights = {}
+      if ttl_data[0][@osf_map['isPublic']][0]['@value'] == 'true'
+        rights['read-groups'] = ['public']
+      end
+      rights
+    end
+
+    # sets the creator- needs to read another ttl for the User data
+    # only contrubutors with isBibliographic true are considered
+    def map_creator
+      creator = []
+      ttl_data[0][@osf_map['hasContributor']].each do |contributor|
+        # Looping through the primary document and the contributors
+        ttl_data.each do |item|
+          next unless item['@id'] == contributor['@id']
+          if item[@osf_map['isBibliographic']][0]['@value'] == 'true'
+            creator.push map_user_from_ttl(item[@osf_map['hasUser']][0]['@id'])
+          end
+        end
+      end
+      creator
+    end
+
+    # read user ttl file, extract User's full name
+    def map_user_from_ttl(file_subpath)
+      user_ttl_data = ttl_from_targz(File.basename(file_subpath))
+      user_ttl_data[0][@osf_map['hasFullName']][0]['@value']
+    end
+  end
+end