rof 1.0.7 → 1.2.0

data/lib/rof/translators.rb

@@ -0,0 +1,23 @@
+Dir.glob(File.expand_path('../translators/*.rb', __FILE__)).each do |filename|
+  require filename
+end
+
+module ROF
+  # A namespace for organizing translating classes. A translating class is responsible for
+  # converting from one format to another format (e.g. CSV to ROF).
+  #
+  # @see ROF::Translator
+  # @see ROF::Translators::CsvToRof
+  # @see ROF::Translators::FedoraToRof
+  # @see ROF::Translators::OsfToRof
+  module Translators
+    # @api public
+    # @param [String] csv_contents - in the form of a CSV
+    # @param [Hash] config - Hash with symbols for keys
+    # @return [Hash] in ROF format
+    # @see ROF::Translators::CsvToRof for full details
+    def self.csv_to_rof(csv_contents, config = {})
+      CsvToRof.call(csv_contents, config)
+    end
+  end
+end

data/lib/rof/{translate_csv.rb → translators/csv_to_rof.rb}

@@ -1,7 +1,8 @@
+require 'rof/translator'
 require('csv')
 require('json')
 
-module ROF
+module ROF::Translators
   # Turn a CSV file into a ROF file.
   #
   # pass in the contents of the CSV file.
@@ -31,7 +32,7 @@ module ROF
   # with the previous work translated into ROF. This will allow a work to have
   # attached files with different access permissions, owners, etc...
   # Any extra files are appended to the file list for the work.
-  class TranslateCSV
+  class CsvToRof < ROF::Translator
     class MissingOwnerOrType < RuntimeError
     end
 
@@ -45,7 +46,7 @@ module ROF
     class NoPriorWork < RuntimeError
     end
 
-    def self.run(csv_contents)
+    def self.call(csv_contents, config = {})
       first_line = nil
       rof_contents = []
       previous_work = nil

data/lib/rof/translators/fedora_to_rof.rb

@@ -0,0 +1,244 @@
+require 'json'
+require 'rexml/document'
+require 'rdf/ntriples'
+require 'rdf/rdfxml'
+require 'rubydora'
+require 'rof/translator'
+
+module ROF
+  module Translators
+    # Responsible for translating Fedora PIDs to ROF objects
+    class FedoraToRof < ROF::Translator
+      # @param [Array] pids - Fedora PIDs
+      # @param [Hash] config - Hash with symbol keys
+      # @option config [Hash] :fedora_connection_information - The Hash that contains the connection information for Fedora
+      # @return [Hash] The ROF representation of teh Fedora objects
+      # @see Rubydora.connect
+      def self.call(pids, config = {})
+        new(pids, config).to_rof
+      end
+
+      def initialize(pids, config = {})
+        @pids = pids
+        @fedora_connection_information = config.fetch(:fedora_connection_information)
+        @config = config
+        connect_to_fedora!
+      end
+      attr_reader :pids, :fedora_connection_information, :config, :connection
+
+      private
+        def connect_to_fedora!
+          @connection = Rubydora.connect(fedora_connection_information)
+        end
+      public
+
+      def to_rof
+        # wrap the objects inside a JSON list
+        pids.map do |pid|
+          PidToRofElement.new(pid, connection, config).convert
+        end
+      end
+
+      # Responsible for converting a single PID to an ROF Element
+      class PidToRofElement
+        def initialize(pid, connection, config)
+          @pid = pid
+          @connection = connection
+          @config = config
+          @fedora_info = { 'pid' => pid, 'type' => 'fobject' }
+          @fedora_object = connection.find(pid)
+        end
+        attr_reader :pid, :config, :fedora_object
+
+        # Given a rubydora object, extract what we need
+        # to create our ROF object in an associative array
+        #
+        def convert
+          @fedora_info['af-model'] = setModel
+          # iterate through the data streams that are present.
+          # use reflection to call appropriate method for each
+          fedora_object.datastreams.each do |dsname, ds|
+            method_name = DATASTREAM_NAME_TO_METHOD_MAP.fetch(dsname) { :default_datastream_conversion }
+            send(method_name, dsname, ds)
+          end
+          @fedora_info
+        end
+
+        DATASTREAM_NAME_TO_METHOD_MAP = {
+          'DC'               => :skip_datastream,
+          'RELS-EXT'         => :convert_rels_ext,
+          'rightsMetadata'   => :convert_rights_metadata,
+          'properties'       => :default_datastream_conversion,
+          'content'          => :default_datastream_conversion,
+          'descMetadata'     => :convert_desc_metadata,
+          'bendo-item'       => :default_datastream_conversion,
+          'characterization' => :default_datastream_conversion,
+          'thumbnail'        => :default_datastream_conversion
+        }
+
+        private
+
+        def default_datastream_conversion(dsname, ds)
+          # dump generic datastream
+          meta = create_meta(ds)
+          @fedora_info["#{dsname}-meta"] = meta unless meta.empty?
+
+          # if content is short < X bytes and valid utf-8, save as string
+          # if content is > X bytes or is not utf-8, save as file only if config option is given
+          content = ds.datastream_content
+          if content.length <= 1024 || config['inline']
+            # this downloads the contents of the datastream into memory
+            content_string = content.to_s.force_encoding('UTF-8')
+            if content_string.valid_encoding?
+              @fedora_info[dsname] = content_string
+              return # we're done! move on to next datastream
+            end
+            # not utf-8, so keep going and see if download option was given
+          end
+          return unless config['download']
+          # download option was given, so save this datastream as a file
+          fname = "#{@fedora_info['pid']}-#{dsname}"
+          abspath = File.join(config['download_path'], fname)
+          @fedora_info["#{dsname}-file"] = fname
+          if File.file?(config['download_path'])
+            $stderr.puts "Error: --download directory #{config['download_path']} specified is an existing file."
+            exit 1
+          end
+          FileUtils.mkdir_p(config['download_path'])
+          File.open(abspath, 'w') do |f|
+            f.write(content)
+          end
+        end
+
+        def create_meta(ds)
+          result = {}
+
+          label = ds.profile['dsLabel']
+          result['label'] = label unless label.nil? || label == ''
+          result['mime-type'] = ds.profile['dsMIME'] if ds.profile['dsMIME'] != 'text/plain'
+          # TODO(dbrower): make sure this is working as intended
+          if %w(R E).include?(ds.profile['dsControlGroup'])
+            s = result['URL'] = ds.profile['dsLocation']
+            s = s.sub(config['bendo'], 'bendo:') if config['bendo']
+            result['URL'] = s
+          end
+          result
+        end
+
+        # set fedora_indo['af-model']
+        #
+        def setModel
+          # only keep info:fedora/afmodel:XXXXX
+          models = fedora_object.profile['objModels'].map do |model|
+            Regexp.last_match(1) if model =~ /^info:fedora\/afmodel:(.*)/
+          end.compact
+          models[0]
+        end
+
+        # The methods below are called if the like-named datastream exists in fedora
+
+        def skip_datastream(*)
+        end
+
+        # set metadata
+        #
+        def convert_desc_metadata(_dsname, ds)
+          # desMetadata is encoded in ntriples, convert to JSON-LD using our special context
+          graph = RDF::Graph.new
+          data = ds.datastream_content
+          # force utf-8 encoding. fedora does not store the encoding, so it defaults to ASCII-8BIT
+          # see https://github.com/ruby-rdf/rdf/issues/142
+          data.force_encoding('utf-8')
+          graph.from_ntriples(data, format: :ntriples)
+          JSON::LD::API.fromRdf(graph) do |expanded|
+            result = JSON::LD::API.compact(expanded, RdfContext)
+            @fedora_info['metadata'] = result
+          end
+        end
+
+        # set rights
+        #
+        def convert_rights_metadata(_dsname, ds)
+          # rights is an XML document
+          # the access array may have read or edit elements
+          # each of these elements may contain group or person elements
+          xml_doc = REXML::Document.new(ds.datastream_content)
+
+          rights_array = {}
+
+          root = xml_doc.root
+
+          # check for optional embargo date - set if present
+          this_embargo = root.elements['embargo']
+          rights_array['embargo-date'] = this_embargo.elements['machine'].elements['date'][0] if Utility.has_embargo_date?(this_embargo)
+
+          %w(read edit).each do |access|
+            this_access = root.elements["//access[@type=\'#{access}\']"]
+
+            next if this_access.nil?
+
+            unless this_access.elements['machine'].elements['group'].nil?
+              group_array = []
+              this_access.elements['machine'].elements['group'].each do |this_group|
+                group_array << this_group
+              end
+              rights_array["#{access}-groups"] = group_array
+            end
+
+            next if this_access.elements['machine'].elements['person'].nil?
+            person_array = []
+
+            this_access.elements['machine'].elements['person'].each do |this_person|
+              person_array << this_person
+            end
+            rights_array[access.to_s] = person_array
+          end
+
+          @fedora_info['rights'] = rights_array
+        end
+
+        def convert_rels_ext(_dsname, ds)
+          # RELS-EXT is RDF-XML - parse it
+          ctx = ROF::RelsExtRefContext.dup
+          ctx.delete('@base') # @base causes problems when converting TO json-ld (it is = "info:/fedora") but info is not a namespace
+          graph = RDF::Graph.new
+          graph.from_rdfxml(ds.datastream_content)
+          result = nil
+          JSON::LD::API.fromRdf(graph) do |expanded|
+            result = JSON::LD::API.compact(expanded, ctx)
+          end
+          # now strip the info:fedora/ prefix from the URIs
+          strip_info_fedora(result)
+          # remove extra items
+          result.delete('hasModel')
+          @fedora_info['rels-ext'] = result
+        end
+
+        private
+
+        def strip_info_fedora(rels_ext)
+          rels_ext.each do |relation, targets|
+            next if relation == '@context'
+            if targets.is_a?(Hash)
+              strip_info_fedora(targets)
+              next
+            end
+            targets = [targets] if targets.is_a?(String)
+            targets.map! do |target|
+              if target.is_a?(Hash)
+                strip_info_fedora(target)
+              else
+                target.sub('info:fedora/', '')
+              end
+            end
+            # some single strings cannot be arrays in json-ld, so convert back
+            # this shouldn't cause any problems with items that began as arrays
+            targets = targets[0] if targets.length == 1
+            rels_ext[relation] = targets
+          end
+        end
+      end
+      private_constant :PidToRofElement
+    end
+  end
+end

data/lib/rof/translators/jsonld_to_rof.rb

@@ -0,0 +1,112 @@
+require 'rof/rdf_context'
+require 'active_support/core_ext/array/wrap'
+require 'rof/translators/jsonld_to_rof/statement_handler'
+require 'rof/translators/jsonld_to_rof/predicate_handler'
+require 'rof/translators/jsonld_to_rof/accumulator'
+
+module ROF
+  module Translators
+    # @api public
+    #
+    # Responsible for converting JSON LD into an ROF Hash via registered URI maps.
+    #
+    # @note Some predicates require explicit mapping where as others have an assumed mapping. At present all URLs for @context of JSON-LD documents must be registered.
+    #
+    # @see ROF::Translators::JsonldToRof.call for details on how the JSON-LD is converted
+    # @see ROF::Translators::PredicateHandler.register for details on how Predicate URI's are mapped to nodes in the ROF document.
+    # @see ROF::Translators::JsonldToRof::PredicateHandler
+    # @see ROF::Translators::JsonldToRof::StatementHandler
+    module JsonldToRof
+      PredicateHandler.register('http://purl.org/ontology/bibo/') do |handler|
+        handler.namespace_prefix('bibo:')
+        handler.within(['metadata'])
+      end
+      PredicateHandler.register('info:fedora/fedora-system:def/relations-external') do |handler|
+        handler.map('#isMemberOfCollection', to: ['rels-ext', 'isMemberOfCollection'])
+      end
+      PredicateHandler.register('http://id.loc.gov/vocabulary/relators/') do |handler|
+        handler.namespace_prefix('mrel:')
+        handler.within(['metadata'])
+      end
+      PredicateHandler.register('http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#') do |handler|
+        handler.namespace_prefix('ebucore:')
+        handler.within(['metadata'])
+      end
+
+      PredicateHandler.register('https://library.nd.edu/ns/terms/') do |handler|
+        handler.map('accessEdit', to: ['rights', 'edit'])
+        handler.map('accessRead', to: ['rights', 'read'])
+        handler.map('accessEditGroup', to: ['rights', 'edit-groups'])
+        handler.map('accessReadGroup', to: ['rights', 'read-groups'])
+        handler.map('accessEmbargoDate', to: ['rights', 'embargo-date'])
+        handler.map('afmodel', to: ["af-model"])
+        handler.map('bendoitem', to: ["bendo-item"])
+        handler.map('depositor') do |object, accumulator|
+          accumulator.register_properties('depositor', object)
+        end
+        handler.map('owner') do |object, accumulator|
+          accumulator.register_properties('owner', object)
+        end
+        handler.map('representativeFile') do |object, accumulator|
+          accumulator.register_properties('representative', object)
+        end
+      end
+
+      PredicateHandler.register('http://purl.org/dc/terms/') do |handler|
+        handler.namespace_prefix('dc:')
+        handler.within(['metadata'])
+        handler.map('contributor', to: ['metadata', 'dc:contributor', 'dc:contributor'], force: true)
+      end
+
+      PredicateHandler.register('http://projecthydra.org/ns/relations#') do |handler|
+        handler.map('hasEditor', to: ['rels-ext', 'hydramata-rel:hasEditor'])
+        # We need to map the hasEditorGroup predicate to two different locations in the ROF
+        handler.map('hasEditorGroup', to: ['rels-ext', 'hydramata-rel:hasEditorGroup'], force: true)
+        handler.map('hasEditorGroup', to: ['rights', 'edit-groups'], force: true)
+      end
+
+      PredicateHandler.register('http://www.ndltd.org/standards/metadata/etdms/1.1/') do |handler|
+        handler.within(['metadata', 'ms:degree'])
+        handler.namespace_prefix('ms:')
+        handler.map('role', to: ['metadata', 'dc:contributor', 'ms:role'], force: true)
+      end
+
+      # @api public
+      #
+      # Convert's the given JSON-LD into an ROF document that can be used to batch ingest into Fedora.
+      #
+      # @param [Array<Hash>, Hash] jsonld - a Hash of JSON-LD data or an Array of JSON-LD Hashes
+      # @param [Hash] config (included to conform to the loose interface of translators)
+      # @return [Array<Hash>] An ROF document
+      def self.call(jsonld, config)
+        Array.wrap(jsonld).map! do |element|
+          Element.new(element).to_rof
+        end
+      end
+
+      # A single top-level element of a JSON-LD document
+      class Element
+        def initialize(element)
+          @element = element
+        end
+
+        def to_rof
+          @accumulator = Accumulator.new(base_rof)
+          JSON::LD::API.toRdf(element) do |statement|
+            StatementHandler.call(statement, accumulator)
+          end
+          @accumulator.to_rof
+        end
+
+        private
+
+        attr_reader :element, :accumulator
+
+        def base_rof
+          { "type" => "fobject", "metadata" => { "@context" => ROF::RdfContext }, "rels-ext" => { "@context" => ROF::RelsExtRefContext } }
+        end
+      end
+      private_constant :Element
+    end
+  end
+end

data/lib/rof/translators/jsonld_to_rof/accumulator.rb

@@ -0,0 +1,175 @@
+require 'active_support/core_ext/array/wrap'
+require 'active_support/core_ext/module/delegation'
+require 'active_support/core_ext/object/deep_dup'
+
+module ROF
+  module Translators
+    module JsonldToRof
+      # @api pubilc
+      # The accumulator is a "passive" object. Things happen to it. All in the name of building the
+      # hash that is ROF.
+      #
+      # @note The accumulator is only for one PID. See [ROF::Translators::JsonldToRof::Accumulator#add_pid]
+      class Accumulator
+        # @param [Hash] initial_rof - The base ROF document to which we will be adding elements.
+        def initialize(initial_rof = {})
+          @rof = initial_rof
+          @blank_nodes = {}
+          @blank_node_locations = {}
+        end
+
+        # @api public
+        # @return [Hash]
+        def to_rof
+          rof = @rof.deep_dup
+          expand_blank_node_locations(rof)
+          append_properties_to(rof)
+          rof
+        end
+
+        private
+
+        # The antics of the blank node! See the specs for blank nodes to see the expected behavior.
+        def expand_blank_node_locations(rof)
+          @blank_node_locations.each_pair do |node, locations|
+            locations.each_pair do |location, key_value_pairs|
+              data = rof
+              location[0..-2].each do |slug|
+                data[slug] ||= {}
+                data = data[slug]
+              end
+
+              # We may encounter a shallow map, if so we need for it to behave differently
+              slug = location[-1]
+              if slug
+                data[slug] ||= []
+                hash = {}
+              else
+                hash = data
+              end
+              Array.wrap(key_value_pairs).each do |key_value|
+                key_value.each_pair do |key, value|
+                  hash[key] ||= []
+                  hash[key] += Array.wrap(value)
+                end
+              end
+              data[slug] << hash if slug
+            end
+          end
+        end
+
+        def append_properties_to(rof)
+          return rof unless @properties
+          rof['properties-meta'] = { "mime-type" => "text/xml" }
+          xml = '<fields>'
+          @properties.each do |node_name, object|
+            xml += "<#{node_name}>#{object}</#{node_name}>"
+          end
+          xml += '</fields>'
+          rof['properties'] = xml
+          rof
+        end
+
+        public
+
+        # @api public
+        # @param [String] node_name - the XML node's name (e.g. <node_name>node_value</node_name>)
+        # @param [String] node_value - the XML element's value
+        # @return [Array] of given node_name and node_value
+        def register_properties(node_name, node_value)
+          @properties ||= []
+          @properties << [node_name, coerce_object_to_string(node_value)]
+          [node_name, node_value]
+        end
+
+        class PidAlreadySetError < RuntimeError
+        end
+
+        # @api public
+        # @param [RDF::Statement] statement
+        # @return [RDF::Statement]
+        def add_blank_node(statement)
+          @blank_nodes[statement.subject] ||= {}
+          @blank_nodes[statement.subject][statement.predicate] ||= []
+          @blank_nodes[statement.subject][statement.predicate] << statement.object
+          statement
+        end
+
+        # @api public
+        # @param [RDF::Subject] subject - Fetch the corresponding blank node that was added
+        # @return [RDF::Statement]
+        # @raise [KeyError] when the subject has not previosly been added
+        # @see #add_blank_node
+        def fetch_blank_node(subject)
+          @blank_nodes.fetch(subject)
+        end
+
+        # @api public
+        # @param [String] pid - an identifier
+        # @return [String] pid
+        # @raise PidAlreadySetError - if you attempted to a different PID
+        def add_pid(pid)
+          pid = coerce_object_to_string(pid)
+          if @rof.key?('pid')
+            if @rof['pid'] != pid
+              raise PidAlreadySetError, "Attempted to set pid=#{pid}, but it is already set to #{@rof['pid']}"
+            end
+          else
+            @rof['pid'] = pid
+          end
+          pid
+        end
+
+        # @api public
+        # @param [Array<String>, String] location - a list of nested hash keys (or a single string)
+        # @param [String] value - a translated value for the original RDF Statement
+        # @param [false, RDF::Node] blank_node
+        # @return [Array] location, value
+        def add_predicate_location_and_value(location, value, blank_node = false)
+          # Because I am making transformation on the location via #shift method, I need a duplication.
+          location = Array.wrap(location)
+          if location == ['pid']
+            return add_pid(value)
+          end
+          if blank_node
+            add_predicate_location_and_value_direct_for_blank_node(location, value, blank_node)
+          else
+            add_predicate_location_and_value_direct_for_non_blank_node(location, value)
+          end
+          [location, value]
+        end
+
+        def add_predicate_location_and_value_direct_for_blank_node(location, value, blank_node)
+          fetch_blank_node(blank_node) # Ensure the node exists
+          @blank_node_locations[blank_node] ||= {}
+          @blank_node_locations[blank_node][location[0..-2]] ||= []
+          @blank_node_locations[blank_node][location[0..-2]] << { location[-1] => Array.wrap(coerce_object_to_string(value)) }
+        end
+
+        def add_predicate_location_and_value_direct_for_non_blank_node(location, value)
+          data = @rof
+          location[0..-2].each do |slug|
+            data[slug] ||= {}
+            data = data[slug]
+          end
+          slug = location[-1]
+          data[slug] ||= []
+          data[slug] << coerce_object_to_string(value)
+        end
+
+        private
+
+        def coerce_object_to_string(object)
+          return object if object.nil?
+          if object.to_s =~ %r{https?://curate.nd.edu/show/([^\\]+)/?}
+            return "und:#{$1}"
+          elsif object.respond_to?(:value)
+            return object.value
+          else
+            object
+          end
+        end
+      end
+    end
+  end
+end