rof 0.0.1.pre → 1.0.4

data/labels.md

@@ -0,0 +1,111 @@
+# Pid Assigner Filter (aka Labels)
+
+This filter will assign pids to objects which don't have one, and provides a labeling system so items without pids and still be linked together using RELS-EXT relationships.
+
+More thought is needed on how to provide linking in RDF since RDF requires a full URI for subjects and objects.
+
+## Parameters
+
+The filter requires the following parameters to work
+
+ - A namespace to use
+ - A server and pool of a noids service to use
+
+## Output
+
+The service does two things: it assigns identifiers to objects, and it provides identifiers for linking in the RELS-EXT section.
+The filter uses the following rules to assign identifiers to objects:
+
+1. Any item not of type `fobject` is skipped.
+
+1. Any item of type `fobject` which does not have a `pid` field will be assigned one using the provided namespace and an identifier generated by the noids service.
+
+1. Any item of type `fobject` which has a label in its `pid` field will be assigned an pid using the provided namespace and an identifier generated by the noids service. The label will be **defined** to be the generated identifier.
+
+1. Any item of type `fobject` which otherwise has a `pid` is skipped.
+
+The filter uses the following rules to provide for linking in the RELS-EXT section:
+
+1. Any item not of type `fobject` is skipped.
+
+1. Any item of type `fobject` with no `rels-ext` section is skipped.
+
+1. Any item of type `fobject` with a `rels-ext` section has all identifiers which are labels **substituted** with the pid assigned to the label. It is an error to use a label which cannot be resolved. See Resolution.
+
+## Labels
+
+A label has the same textual form whether it is used to assign a pid or to reference an assigned pid.
+A label begins with a dollar sign and an open parenthesis, contains a label name, and ends with a close parenthesis.
+For example, `$(label-name)`
+A label name may contain any character except close parenthesis `)`.
+Labels which end with the following strings are reserved:
+
+    -noid
+    -ns
+    -info
+
+These suffixes are reserved for future use as providing alternate forms of an identifier.
+
+I see the replacement of labels in text blocks as a possible way to adapt this to use in the `descMetadata` field, e.g.
+
+    https://curate.nd.edu/show/$(label-noid)
+
+## Resolution
+
+Identifiers do not need to be defined before they are referenced; they merely need to be defined eventually.
+This property allows for complicated object constellations such as cycles, and even self-references.
+It also makes it easier to create input files since ROF creators do not need to make sure objects which define a label are listed before any references to that label.
+
+## Example
+
+The following code shows how a list of objects with labels is transformed.
+The input is
+
+```
+[
+  {
+    "type" : "fobject",
+    "pid"  : "$(first)"
+  },
+  {
+    "type" : "fobject",
+    "rels-ext" : {
+        "memberOf" : ["$(first)", "$(second)"]
+    }
+  },
+  {
+    "type" : "fobject",
+    "pid"  : "$(second)"
+  }
+]
+```
+
+Results in the following output after applying the filter.
+
+```
+[
+  {
+    "type": "fobject",
+    "pid": "temp:001"
+  },
+  {
+    "type": "fobject",
+    "rels-ext": {
+      "memberOf": [
+        "temp:001",
+        "temp:003"
+      ]
+    },
+    "pid": "temp:002"
+  },
+  {
+    "type": "fobject",
+    "pid": "temp:003"
+  }
+]
+```
+
+
+
+
+

data/lib/rof.rb

@@ -1,5 +1,24 @@
+require "rof/ingest"
 require "rof/version"
+require "rof/cli"
+require "rof/access"
+require "rof/collection"
+require "rof/utility"
+require "rof/rdf_context"
+require "rof/translate_csv"
+require "rof/filters/date_stamp"
+require "rof/filters/file_to_url"
+require "rof/filters/label"
+require "rof/filters/work"
+require "rof/filters/bendo"
 
 module ROF
+end
 
-end
+# work around Rubydora expecting a logger
+unless defined?(logger)
+  def logger
+    require 'logger'
+    @logger ||= Logger.new(STDOUT)
+  end
+end

data/lib/rof/access.rb

@@ -0,0 +1,57 @@
+module ROF
+  # provide translation between access strings and the ROF access hash
+  # e.g. ("public", owner=dbrower) --> {read-groups: "public", edit: "dbrower"}
+  class Access
+    class DecodeError < RuntimeError
+    end
+
+    # convert from a string to a hash
+    def self.decode(access_string, owner=nil)
+      result = {}
+      access_string.split(";").each do |clause|
+        t = self.decode_clause(clause, owner)
+        t.each do |k,v|
+          if v.is_a?(Array)
+            result[k] = (result.fetch(k, []) + v).uniq
+          else
+            result[k] = v
+          end
+        end
+      end
+
+      result
+    end
+
+    # convert from a hash to a string
+    # simple because we do not try to recover "public", et al.
+    def self.encode(access_hash)
+      result = []
+      access_hash.each do |k,v|
+        xk = k.gsub("-groups", "group").gsub("embargo-date","embargo")
+        xv = v.join(',') if v.is_a?(Array)
+        result << "#{xk}=#{xv}"
+      end
+      result.join(";")
+    end
+
+    def self.decode_clause(access, owner)
+      case access
+      when "public"
+        {"read-groups" => ["public"], "edit" => [owner]}
+      when "restricted"
+        {"read-groups" => ["registered"], "edit" => [owner]}
+      when "private"
+        {"edit" => [owner]}
+      when /^embargo=(.+)/
+        {"embargo-date" => $1}
+      when /^(read|readgroup|edit|editgroup|discover|discovergroup)=(.+)/
+        which = $1
+        who = $2.split(",")
+        xwhich = which.gsub("group", "-groups")
+        Hash[xwhich, who]
+      else
+        raise DecodeError
+      end
+    end
+  end
+end

data/lib/rof/cli.rb

@@ -0,0 +1,122 @@
+require 'benchmark'
+require 'rof/compare_rof'
+require 'json'
+require 'rubydora'
+require 'rof/ingest'
+require 'rof/collection'
+require 'rof/get_from_fedora'
+require 'rof/osf_to_rof'
+module ROF
+  module CLI
+    # Ingest the file `fname` that is a level 0 rof file. It may contain any
+    # number of fedora objects; they will be delt with in the order they appear
+    # in the file. Any external files (except fname) are searched for using the
+    # `search_path` array of directories. If `fedora` is present, it is a hash
+    # having the keys `url`, `user`, and `password`. Omitting `fedora` has the
+    # effect of verifying the format of `fname`.
+    #
+    # All output is sent to `outfile`.
+    #
+    # Returns the number of errors.
+    def self.ingest_file(fname, search_paths = [], outfile = STDOUT, fedora = nil, bendo = nil)
+      items = load_items_from_file(fname, outfile)
+      ingest_array(items, search_paths, outfile, fedora, bendo)
+    end
+
+    def self.ingest_array(items, search_paths = [], outfile = STDOUT, fedora = nil, bendo = nil)
+      need_close = false
+      if outfile.nil?
+        outfile = File.open('/dev/null', 'w')
+        need_close = true
+      end
+      fedora = Rubydora.connect(fedora) if fedora
+      item_count = 1
+      error_count = 0
+      verb = fedora.nil? ? 'Verifying' : 'Ingesting'
+      overall_benchmark = Benchmark.measure do
+        items.each do |item|
+          begin
+            outfile.write("#{item_count}. #{verb} #{item['pid']} ...")
+            item_count += 1
+            individual_benchmark = Benchmark.measure do
+              ROF.Ingest(item, fedora, search_paths, bendo)
+            end
+            outfile.write("ok. %0.3fs\n" % individual_benchmark.real)
+          rescue Exception => e
+            error_count += 1
+            outfile.write("error. #{e}\n")
+            # TODO(dbrower): add option to toggle displaying backtraces
+            if e.backtrace
+              outfile.write(e.backtrace.join("\n\t"))
+              outfile.write("\n")
+            end
+          end
+        end
+      end
+      outfile.write("Total time %0.3fs\n" % overall_benchmark.real)
+      outfile.write("#{error_count} errors\n")
+      error_count
+    ensure
+      outfile.close if outfile && need_close
+    end
+
+    def self.filter_file(filter, fname, outfile = STDOUT)
+      items = load_items_from_file(fname, STDERR)
+      filter_array(filter, items, fname, outfile)
+    end
+
+    def self.filter_array(filter, items, fname, outfile = STDOUT)
+      # filter will transform the items array in place
+      result = filter.process(items, fname)
+      outfile.write(JSON.pretty_generate(result))
+    end
+
+    # retrieve fedora object and convert to ROF
+    def self.convert_to_rof(pids, fedora = nil, outfile = STDOUT, config = {})
+      need_close = false
+      # use outfile is_a String
+      if outfile.is_a?(String)
+        outfile = File.open(outfile, 'w')
+        need_close = true
+      end
+
+      # wrap the objects inside a JSON list
+      result = []
+      pids.each do |pid|
+        result << ROF::FedoraToRof.GetFromFedora(pid, fedora, config)
+      end
+      outfile.write(JSON.pretty_generate(result))
+    ensure
+      outfile.close if outfile && need_close
+    end
+
+    # convert OSF archive tar.gz to rof file
+    def self.osf_to_rof(config, outfile = STDOUT)
+      osf_projects = load_items_from_file(config['project_file'], outfile) if config.key?('project_file')
+      rof_data = ROF::OsfToRof.osf_to_rof(config, osf_projects[0])
+      outfile.write(JSON.pretty_generate(rof_data))
+    end
+
+    # compare two rofs
+    def self.compare_files(file1, file2, outfile = STDOUT, _fedora, _bendo)
+      fedora_rof = load_items_from_file(file1, outfile)
+      bendo_rof =  load_items_from_file(file2, outfile)
+
+      ROF::CompareRof.fedora_vs_bendo(fedora_rof, bendo_rof, outfile)
+    end
+
+    protected
+
+    def self.load_items_from_file(fname, outfile)
+      items = nil
+      File.open(fname, 'r:UTF-8') do |f|
+        items = JSON.parse(f.read)
+      end
+      items = [items] unless items.is_a? Array
+      items
+    rescue JSON::ParserError => e
+      outfile.puts("Error reading #{fname}:#{e}")
+      exit!(1)
+    end
+  end
+end

data/lib/rof/collection.rb

@@ -0,0 +1,109 @@
+require 'mime-types'
+
+module ROF
+  # Called from ROF::Work.process_one_work
+  # Can assume type fobject, af-model Collection
+  class Collection
+    class NoFile < RuntimeError
+    end
+    def self.process_one_collection(input_obj, utility)
+      # set the required fields
+      result = set_required_fields(input_obj, utility)
+      result = make_images(result, input_obj, utility)
+      result
+    end
+
+    # Set the fields that must be there
+    def self.set_required_fields(obj, utility)
+      result = {}
+      result['type'] = 'fobject'
+      result['af-model'] = 'Collection'
+      result['rights'] = obj['rights']
+      result['metadata'] = obj['metadata']
+      result['pid'] = obj.fetch('pid', utility.next_label)
+      result['rels-ext'] = obj.fetch('rels-ext', {})
+      result['properties'] = ROF::Utility.prop_ds(obj['owner'])
+      result['properties-meta'] = { 'mime-type' => 'text/xml' }
+      result
+    end
+
+    # If collection included a file, create launch image and thumbnaile
+    def self.make_images(subtotal, obj, utility)
+      return subtotal if obj['files'].nil?
+
+      # verify source image is present in job dir
+      image_source = File.join(utility.workdir, obj['files'][0])
+
+      # attempt to create a launch page image and thumbnail
+      # exit if either fails
+      unless File.exist?(image_source)
+        STDERR.print("ROF:Collection.make_images: file  ", image_source, " does not exist.\n")
+        raise NoFile
+      end
+      create_images(subtotal, image_source)
+    end
+
+    def self.create_images(obj, image_source)
+      launch_img = make_launch(image_source)
+      thumb_img = make_thumb(image_source)
+      raise NoFile if launch_img.nil? || thumb_img.nil?
+      obj['content-file'] = File.basename(launch_img)
+      obj['content-meta'] = { 'mime-type' => find_file_mime(launch_img) }
+      obj['thumbnail-file'] = File.basename(thumb_img)
+      obj['thumbnail-meta'] = { 'mime-type' => find_file_mime(thumb_img) }
+      obj
+    end
+
+    # make collections launch page image
+    def self.make_launch(src_image)
+      options = ' -resize 350x350 '
+
+      dest_image = mk_dest_img_name(src_image, '-launch')
+      unless run_convert(src_image, dest_image, options)
+        STDERR.print("ROF:Collection.mk_launch: failed on file  ", src_image, ".\n")
+        return nil
+      end
+      dest_image
+    end
+
+    # make thumbnail
+    def self.make_thumb(src_image)
+      options = ' -resize 256x256 '
+
+      dest_image = mk_dest_img_name(src_image, '-thumb')
+      unless run_convert(src_image, dest_image, options)
+        STDERR.print("ROF:Collection.mk_thumb: failed on file  ", src_image, ".\n")
+        return nil
+      end
+      dest_image
+    end
+
+    def self.run_convert(src_image, dest_image, options)
+      command = set_convert_path + ' ' + src_image + options + ' ' + dest_image
+      Kernel.system(command)
+    end
+
+    # figure out where ImageMagick is installed
+    # (assumes brew path on MacOS, binary RPM path on Linux).
+    def self.set_convert_path
+      host_os = RbConfig::CONFIG['sitearch']
+
+      return '/usr/local/bin/convert' if host_os.include? 'darwin'
+      '/usr/bin/convert'
+    end
+
+    # given source image, create destination name for conversion
+    # keep same mime type - use dumb mime type determination
+    def self.mk_dest_img_name(src_img, dest_name)
+      dest_part = src_img.split('.')
+      dest_img = dest_part[0] + dest_name
+      dest_img = dest_img + '.' + dest_part[1] if dest_part.length == 2
+      dest_img
+    end
+
+    # extract file extension and determine mime/type.
+    def self.find_file_mime(filename)
+      MIME::Types.of(filename).first.content_type
+    end
+  end
+end

data/lib/rof/compare_rof.rb

@@ -0,0 +1,92 @@
+require 'json'
+require 'rdf/ntriples'
+require 'rdf/rdfxml'
+require 'rof/rdf_context'
+require 'rdf/isomorphic'
+
+module ROF
+ class CompareRof
+
+    # compare fedora rof to bendo_rof
+    # return true in equivalent, false if not
+    def self.fedora_vs_bendo( fedora_rof, bendo_rof, output)
+
+      error_count = 0
+      # dereferencing an array of one element with [0]. Oh, the horror of it.
+      error_count += compare_rights( fedora_rof[0], bendo_rof[0], output)
+      error_count += compare_rels_ext(fedora_rof[0], bendo_rof[0])
+      error_count += compare_metadata(fedora_rof[0], bendo_rof[0])
+      error_count += compare_everything_else(fedora_rof[0], bendo_rof[0], output)
+      error_count
+    end
+
+    # do rights comparison
+    # return 0 if the same, >0 if different
+    def self.compare_rights( fedora_rof, bendo_rof, output )
+
+      error_count =0
+
+      # Use same comparison scheme on all rights
+      [ 'read' , 'read-groups', 'edit', 'edit-groups', 'edit-users', 'embargo-date'].each do |attribute|
+        error_count += rights_equal(attribute, fedora_rof, bendo_rof)
+        break if error_count != 0
+      end
+
+      error_count
+    end
+
+    # compare array or element for equivalence
+    def self.rights_equal(rights_attr, fedora, bendo)
+      f_rights = fedora.fetch('rights', {}).fetch(rights_attr, [])
+      b_rights = bendo.fetch('rights', {}).fetch(rights_attr, [])
+
+      f_rights = f_rights.sort if f_rights.respond_to?(:"sort")
+      b_rights = b_rights.sort if b_rights.respond_to?(:"sort")
+
+      return 0 if f_rights == b_rights
+      1
+    end
+
+    # convert RELS-EXT sections to RDF::graph and compater w/ rdf-isomorphic
+    def self.compare_rels_ext(fedora, bendo)
+      error_count = 0
+      bendo_rdf = jsonld_to_rdf(bendo['rels-ext'], ROF::RelsExtRefContext)
+      fedora_rdf = jsonld_to_rdf(fedora['rels-ext'], ROF::RelsExtRefContext)
+      error_count +=1 if ! bendo_rdf.isomorphic_with? fedora_rdf
+      error_count
+    end
+    
+    def self.jsonld_to_rdf(doc, default_context)
+      doc["@context"] = default_context unless doc.has_key?("@context")
+      RDF::Graph.new << JSON::LD::API.toRdf(doc)
+    end
+
+    # convert metadata sections to RDF::graph and compater w/ rdf-isomorphic
+    def self.compare_metadata(fedora, bendo)
+      error_count = 0
+      bendo_rdf = jsonld_to_rdf(bendo['metadata'], ROF::RdfContext)
+      fedora_rdf = jsonld_to_rdf(fedora['metadata'], ROF::RdfContext)
+      error_count +=1 if ! bendo_rdf.isomorphic_with? fedora_rdf
+      error_count
+    end
+
+    # compare what remains
+    def self.compare_everything_else( fedora, bendo, output)
+      error_count =0
+      fedora = remove_others(fedora)
+      bendo = remove_others(bendo)
+      # comparsion using builtin equivalency operation
+      error_count = 1 if bendo != fedora
+      error_count
+    end
+
+    # remove elements we've dealt with already
+    def self.remove_others( rof_object)
+      rof_object.delete('rights')
+      rof_object.delete('rels-ext')
+      rof_object.delete('metadata')
+      rof_object.delete('thumbnail-file')
+      rof_object
+    end
+  end
+end