rof 0.0.1.pre → 1.0.4

data/lib/rof/filters/bendo.rb

@@ -0,0 +1,33 @@
+require 'date'
+
+module ROF
+  module Filters
+
+    # If bendo server is set , add it into datasreams that contain an URl referencing bendo
+    #
+    class Bendo
+      def initialize(bendo=nil)
+        @bendo = bendo
+      end
+
+      def process(obj_list, _fname)
+
+        ends_meta = Regexp.new('(.+)-meta')
+
+        # for *-meta objects containing "URL", sub in bendo string if provided
+
+        obj_list.map! do |obj|
+          obj.map do |name, value|
+            if name =~ ends_meta
+              if obj[name]["URL"] && @bendo
+                obj[name]["URL"] = obj[name]["URL"].sub("bendo:",@bendo)
+              end
+            end
+          end
+          # print object
+          obj
+        end
+      end
+    end
+  end
+end

data/lib/rof/filters/date_stamp.rb

@@ -0,0 +1,36 @@
+require 'date'
+
+module ROF
+  module Filters
+    # Set the upload date to be the date given, provided it doesn't already exist.
+    # Also set the date modified to be the date given.
+    # If not given, the date used defaults to the local time on the computer.
+    class DateStamp
+      def initialize(date=nil)
+        @today = date || Date::today
+        @today_s = if @today.is_a?(Date)
+                     @today.strftime('%FZ')
+                   else
+                     @today.to_s
+                   end
+      end
+
+      def process(obj_list, _fname)
+        obj_list.map! do |obj|
+          if obj["metadata"].nil?
+            obj["metadata"] = {
+              "@context" => ROF::RdfContext
+            }
+          end
+          # only save the date submitted if it is not already present
+          if obj["metadata"]["dc:dateSubmitted"].nil?
+            obj["metadata"]["dc:dateSubmitted"] = @today_s
+          end
+          # always update the date modified
+          obj["metadata"]["dc:modified"] = @today_s
+          obj
+        end
+      end
+    end
+  end
+end

data/lib/rof/filters/file_to_url.rb

@@ -0,0 +1,27 @@
+module ROF
+  module Filters
+    # Convert any content datastream files into a bendo URL, and alter the rof
+    # to use the URL and not upload the file to fedora directly. The bendo URL
+    # will only exist for items having a bendo-item id set. The URL generated
+    # supposes the file keeps the same relative path the item originally had in
+    # the rof file.
+    class FileToUrl
+      def initialize()
+      end
+
+      def process(obj_list, _fname)
+        obj_list.map! do |obj|
+          bendo_item = obj['bendo-item']
+          content_file = obj['content-file']
+          if bendo_item && content_file
+            new_meta = obj.fetch('content-meta', {})
+            new_meta['URL'] = "bendo:/item/#{bendo_item}/#{content_file}"
+            obj['content-meta'] = new_meta
+            obj.delete('content-file')
+          end
+          obj
+        end
+      end
+    end
+  end
+end

data/lib/rof/filters/label.rb

@@ -0,0 +1,153 @@
+require 'noids_client'
+
+module ROF
+  module Filters
+    # Class Label locates in-place labels of the form
+    # "$(label_name)" in the ROF file, assigns each
+    # label a pid, then replaces the label with that pid.
+    class Label
+      class MissingLabel < RuntimeError
+      end
+
+      class NoPool < RuntimeError
+      end
+
+      class OutOfIdentifiers < RuntimeError
+      end
+
+      # Create a new label assigner and resolver. The source of identifiers
+      # is given using options.
+      # Use :noid_server and :pool_name to connect to an external noid server.
+      # Use :id_list to pass in a ruby object responding to #shift and #empty? to generate
+      # ids. This is usually a list, to facilitate testing.
+      #
+      # If prefix is not nil, then "#{prefix}:" is prepended to
+      # every identifier.
+      def initialize(prefix, options)
+        @id_list =  case
+                    when options[:id_list]
+                      options[:id_list]
+                    when options[:noid_server]
+                      NoidsPool.new(options[:noid_server], options[:pool_name])
+                    else
+                      raise NoPool
+                    end
+        @prefix = "#{prefix}:" if prefix
+        # The first match group in the RE provides the label name
+        @label_re = /\$\(([^)]+)\)/
+      end
+
+      # mutate obj_list by assigning labels and resolving labels where needed
+      # Every fobject will be assigned an pid and a bendo_item
+      def process(obj_list, _fname)
+        labels = {}
+
+        # Use two passes. First assign ids, and then resolve labels
+        # Do this since labels can be referenced before being defined
+
+        # Assign pids to each fobject. If we find any labels in the pid field, then
+        # record a mapping of label => pid into the labels hash.
+        obj_list.each do |obj|
+          assign_pid(obj, labels)
+        end
+
+        # now replace any reference labels with the pids we've assigned them
+        obj_list.each do |obj|
+          replace_labels_in_obj(obj, labels)
+        end
+
+        # now assign bendo ids
+        bendo_item = nil
+        obj_list.each do |obj|
+          # for now we just use the first item's pid stripped of any namespaces as the bendo item id
+          if bendo_item.nil?
+            bendo_item = obj['pid'].gsub(/^.*:/, '') unless obj['pid'].nil?
+            next if bendo_item.nil?
+          end
+          # don't touch if a bendo item has already been assigned
+          obj['bendo-item'] = bendo_item if obj['bendo-item'].nil? || obj['bendo-item'] == ''
+        end
+
+        obj_list
+      end
+
+      # assign pids, recording any labels we find.
+      # obj is mutated
+      def assign_pid(obj, labels)
+        return if obj['type'] != 'fobject'
+
+        label = nil
+        unless obj['pid'].nil?
+          label = find_label(obj['pid'])
+          # skip if the "pid" is not a label
+          return if label.nil?
+        end
+        pid = "#{@prefix}#{next_id}"
+        obj['pid'] = pid
+        labels[label] = pid unless label.nil?
+      end
+
+      # replace any label references we find in obj.
+      # obj is mutated
+      def replace_labels_in_obj(obj, labels)
+        return if obj['type'] != 'fobject'
+        obj.each do |k, v|
+          # only force labels to exist if we are looking in the rels-ext
+          obj[k] = replace_labels(v, labels, k == 'rels-ext')
+        end
+      end
+
+      # recurse through obj replacing any labels in strings
+      # with the id in labels, which is a hash.
+      # The relacement is done in place.
+      # Hash keys are not touched (only hash values).
+      # if force is true, labels which don't resolve will raise
+      # a MissingLabel error.
+      def replace_labels(obj, labels, force = false)
+        if obj.is_a?(Array)
+          obj.map! { |x| replace_labels(x, labels, force) }
+        elsif obj.is_a?(Hash)
+          obj.each { |k, v| obj[k] = replace_labels(v, labels, force) }
+          obj
+        elsif obj.is_a?(String)
+          replace_match(obj, labels, force)
+        else
+          obj
+        end
+      end
+
+      # small matching function- uses regular expression
+      def replace_match(obj, labels, force)
+        obj.gsub(@label_re) do |match|
+          pid = labels[Regexp.last_match(1)]
+          raise MissingLabel if pid.nil? && force
+          pid.nil? ? match : pid
+        end
+      end
+
+      def find_label(s)
+        s[@label_re, 1]
+      end
+
+      def next_id
+        raise OutOfIdentifiers if @id_list.empty?
+        @id_list.shift
+      end
+
+      # Encapsulates connection to Noids Server
+      class NoidsPool
+        def initialize(noids_server, pool_name)
+          @pool = NoidsClient::Connection.new(noids_server).get_pool(pool_name)
+        end
+
+        def shift
+          @pool.mint.first
+        end
+
+        def empty?
+          @pool.closed?
+        end
+      end
+    end
+  end
+end

data/lib/rof/filters/work.rb

@@ -0,0 +1,111 @@
+require 'mime-types'
+
+module ROF
+  module Filters
+    # Expand objects of type "Work(-(.+))?" into a
+    # constellation of "fobjects".
+    # Makes a fobject/generic_file for each file
+    # adds a depositor
+    # turns original object into an fobject/$1
+    # and copies the access to each fobject.
+    class Work
+      class NoFile < RuntimeError
+      end
+
+      def initialize
+        @utility = ROF::Utility.new
+      end
+
+      # wade through object list
+      def process(obj_list, filename)
+        @utility.set_workdir(filename)
+        obj_list.map! { |x| process_one_work(x) }
+        obj_list.flatten!
+      end
+
+      # given a single object, return a list (possibly empty) of new objects
+      # to replace the one given
+      def process_one_work(input_obj)
+        model = @utility.decode_work_type(input_obj)
+        return [input_obj] if model.nil?
+        return [ROF::Collection.process_one_collection(input_obj, @utility)] if model == 'Collection'
+
+        main_obj = set_main_obj(input_obj, model)
+
+        result = [main_obj]
+        result = make_thumbnail(result, main_obj, input_obj) unless input_obj['files'].nil?
+        result
+      end
+
+      # make the first file be the representative thumbnail
+      def make_thumbnail(result, main_obj, input_obj)
+        thumb_rep = nil
+        input_obj['files'].each do |finfo|
+          if finfo.is_a?(String)
+            fname = finfo
+            finfo = { 'files' => [fname] }
+          else
+            fname = finfo['files'].first
+            raise NoFile if fname.nil?
+          end
+          finfo['rights'] ||= input_obj['rights']
+          finfo['owner'] ||= input_obj['owner']
+          finfo['bendo-item'] ||= input_obj['bendo-item']
+          finfo['metadata'] ||= {
+            '@context' => ROF::RdfContext
+          }
+          finfo['metadata']['dc:title'] ||= fname
+          mimetype = MIME::Types.of(fname)
+          mimetype = mimetype.empty? ? 'application/octet-stream' : mimetype.first.content_type
+          f_obj = {
+            'type' => 'fobject',
+            'af-model' => 'GenericFile',
+            'pid' => finfo['pid'],
+            'bendo-item' => finfo['bendo-item'],
+            'rights' => finfo['rights'],
+            'properties' => ROF::Utility.prop_ds(finfo['owner']),
+            'properties-meta' => {
+              'mime-type' => 'text/xml'
+            },
+            'rels-ext' => {
+              'isPartOf' => [main_obj['pid']]
+            },
+            'content-file' => fname,
+            'content-meta' => {
+              'label' => fname,
+              'mime-type' => mimetype
+            },
+            'collections' => finfo['collections'],
+            'metadata' => finfo['metadata']
+          }
+          f_obj.delete_if { |_k, v| v.nil? }
+          if thumb_rep.nil?
+            thumb_rep = f_obj['pid']
+            if thumb_rep.nil?
+              thumb_rep = @utility.next_label
+              f_obj['pid'] = thumb_rep
+            end
+            main_obj['properties'] = ROF::Utility.prop_ds(input_obj['owner'], thumb_rep)
+          end
+          result << f_obj
+        end
+        result
+      end
+
+      def set_main_obj(input_obj, model)
+        result = {}
+
+        result['type'] = 'fobject'
+        result['af-model'] = model
+        result['pid'] = input_obj.fetch('pid', @utility.next_label)
+        result['bendo-item'] = input_obj['bendo-item']
+        result['rights'] = input_obj['rights']
+        result['properties'] = ROF::Utility.prop_ds(input_obj['owner'])
+        result['properties-meta'] = { 'mime-type' => 'text/xml' }
+        result['rels-ext'] = input_obj.fetch('rels-ext', {})
+        result['metadata'] = input_obj['metadata']
+        result
+      end
+    end
+  end
+end

data/lib/rof/get_from_fedora.rb

@@ -0,0 +1,196 @@
+require 'json'
+require 'rexml/document'
+require 'rdf/ntriples'
+require 'rdf/rdfxml'
+require 'rubydora'
+
+module ROF
+  class FedoraToRof
+    # connect to fedora and fetch object
+    # returns array of fedora attributes or nil
+    def self.GetFromFedora(pid, fedora, config)
+      @fedora_info = {}
+
+      # Try to connect to fedora, and search for the desired item
+      # If either of these actions fail, handle it, and exit.
+      begin
+        fedora = Rubydora.connect(fedora)
+        doc = fedora.find(pid)
+      rescue StandardError => e
+        puts "Error: #{e}"
+        exit 1
+      end
+
+      # set pid, type
+      @fedora_info['pid'] = pid
+      @fedora_info['type'] = 'fobject'
+
+      readFedora(doc, config)
+
+      @fedora_info
+    end
+
+    # Given a rubydora object, extract what we need
+    # to create our ROF object in an associative array
+    #
+    def self.readFedora(rdora_obj, config)
+      @fedora_info['af-model'] = setModel(rdora_obj)
+      # iterate through the data streams that are present.
+      # use reflection to call appropriate method for each
+      rdora_obj.datastreams.each do |dsname, ds|
+        next if dsname == 'DC'
+        method_key = dsname.sub('-', '')
+        if respond_to?(method_key)
+          send(method_key, ds, config)
+        else
+          # dump generic datastream
+          meta = create_meta(ds, config)
+          @fedora_info["#{dsname}-meta"] = meta unless meta.empty?
+
+          # TODO(dbrower): change dump algorithm:
+          # if content is short < X bytes, save as string
+          # if content is > X bytes, save as file only if config option is given
+          content = ds.datastream_content
+          # NOTE- Entire datastream being downloaded every time.
+          content_string = content.to_s.force_encoding('UTF-8')
+          if (content.length <= 1024 || config['inline']) && content_string.valid_encoding?
+            @fedora_info[dsname] = content_string
+          elsif config['download']
+            fname = "#{@fedora_info['pid']}-#{dsname}"
+            abspath = File.join(config['download_path'], fname)
+            @fedora_info["#{dsname}-file"] = fname
+            if File.file?(config['download_path'])
+              puts "Error: --download directory #{config['download_path']} specified is an existing file."
+              exit 1
+            end
+            FileUtils.mkdir_p(config['download_path'])
+            File.open(abspath, 'w') do |f|
+              f.write(content)
+            end
+          end
+        end
+      end
+    end
+
+    def self.create_meta(ds, config)
+      result = {}
+
+      label = ds.profile['dsLabel']
+      result['label'] = label unless label.nil? || label == ''
+      result['mime-type'] = ds.profile['dsMIME'] if ds.profile['dsMIME'] != 'text/plain'
+      # TODO(dbrower): make sure this is working as intended
+      if %w(R E).include?(ds.profile['dsControlGroup'])
+        s = result['URL'] = ds.profile['dsLocation']
+        s = s.sub(config['bendo'], 'bendo:') if config['bendo']
+        result['URL'] = s
+      end
+      result
+    end
+
+    # set fedora_indo['af-model']
+    #
+    def self.setModel(rdora_obj)
+      # only keep info:fedora/afmodel:XXXXX
+      models = rdora_obj.profile['objModels'].map do |model|
+        Regexp.last_match(1) if model =~ /^info:fedora\/afmodel:(.*)/
+      end.compact
+      models[0]
+    end
+
+    # The methods below are called if the like-named datastream exists in fedora
+
+    # set metadata
+    #
+    def self.descMetadata(ds, _config)
+      # desMetadata is encoded in ntriples, convert to JSON-LD using our special context
+      graph = RDF::Graph.new
+      data = ds.datastream_content
+      # force utf-8 encoding. fedora does not store the encoding, so it defaults to ASCII-8BIT
+      # see https://github.com/ruby-rdf/rdf/issues/142
+      data.force_encoding('utf-8')
+      graph.from_ntriples(data, format: :ntriples)
+      JSON::LD::API.fromRdf(graph) do |expanded|
+        result = JSON::LD::API.compact(expanded, RdfContext)
+        @fedora_info['metadata'] = result
+      end
+    end
+
+    # set rights
+    #
+    def self.rightsMetadata(ds, _config)
+      # rights is an XML document
+      # the access array may have read or edit elements
+      # each of these elements may contain group or person elements
+      xml_doc = REXML::Document.new(ds.datastream_content)
+
+      rights_array = {}
+
+      root = xml_doc.root
+
+      %w(read edit).each do |access|
+        this_access = root.elements["//access[@type=\'#{access}\']"]
+
+        next if this_access.nil?
+
+        unless this_access.elements['machine'].elements['group'].nil?
+          group_array = []
+          this_access.elements['machine'].elements['group'].each do |this_group|
+            group_array << this_group
+          end
+          rights_array["#{access}-groups"] = group_array
+        end
+
+        next if this_access.elements['machine'].elements['person'].nil?
+        person_array = []
+
+        this_access.elements['machine'].elements['person'].each do |this_person|
+          person_array << this_person
+        end
+        rights_array[access.to_s] = person_array
+      end
+
+      @fedora_info['rights'] = rights_array
+    end
+
+    def self.RELSEXT(ds, _config)
+      # RELS-EXT is RDF-XML - parse it
+      ctx = ROF::RelsExtRefContext.dup
+      ctx.delete('@base') # @base causes problems when converting TO json-ld (it is = "info:/fedora") but info is not a namespace
+      graph = RDF::Graph.new
+      graph.from_rdfxml(ds.datastream_content)
+      result = nil
+      JSON::LD::API.fromRdf(graph) do |expanded|
+        result = JSON::LD::API.compact(expanded, ctx)
+      end
+      # now strip the info:fedora/ prefix from the URIs
+      strip_info_fedora(result)
+      # remove extra items
+      result.delete('hasModel')
+      @fedora_info['rels-ext'] = result
+    end
+
+    private
+
+    def self.strip_info_fedora(rels_ext)
+      rels_ext.each do |relation, targets|
+        next if relation == '@context'
+        if targets.is_a?(Hash)
+          strip_info_fedora(targets)
+          next
+        end
+        targets = [targets] if targets.is_a?(String)
+        targets.map! do |target|
+          if target.is_a?(Hash)
+            strip_info_fedora(target)
+          else
+            target.sub('info:fedora/', '')
+          end
+        end
+        # some single strings cannot be arrays in json-ld, so convert back
+        # this shouldn't cause any problems with items that began as arrays
+        targets = targets[0] if targets.length == 1
+        rels_ext[relation] = targets
+      end
+    end
+  end
+end