fsp_harvester 0.1.3 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b34dd6bfa1749e85a0c9a584f6ae8d5ae2181e420b22b383f06571eb9d7d81b1
-  data.tar.gz: 232c3607283e9a633f7b4307672b100e9d57f52d5bcbd31b4b9386a9d025c771
+  metadata.gz: f24e34b5239426a8555e5d893cb3692a1f03442f6b4af3c03c5751c975a7871b
+  data.tar.gz: d372b73eb7693e5a4c9a2f78e20d02b62f3c195ed4185db7532018a45a694570
 SHA512:
-  metadata.gz: f8b8816dcb73253751530f88b8537b42ca7fa51cf8895ef166d37bd6afb1ce310b4bdea66f0acedf7481203ea84b4762b2742389f6a11015f352e6ed979ff1c3
-  data.tar.gz: 15783aa27fa3c494bee0dc2dc50a85f4f6216b3f8f172cbc743eb7aade2e96066e0752491e844af1bcfd5aa268da334d0633b648d936178b68445534fce99b75
+  metadata.gz: e85c8ba90bee37156e8a4d8e98ec0d8c2148ffc86e24ac3faf0adde1146efaafa2333e7e25acf2b5b4d05aa1f2a9a411deb18890175e93bd1b8d0980773e42c2
+  data.tar.gz: f20559315f1b9aff81978600f50743fdee2d3b200eae0dc375ccf267fda90909cb16a6e43686a5efea0f6c583c90c4511c7390702ad7f9e9704f80f829da128b

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    fsp_harvester (0.1.2)
+    fsp_harvester (0.1.6)
       json (~> 2.0)
       linkeddata (~> 3.2)
-      linkheaders-processor (~> 0.1.10)
+      linkheaders-processor (~> 0.1.12)
       metainspector (~> 5.11.2)
       parseconfig (~> 1.1)
       rake (~> 13.0)
@@ -126,7 +126,7 @@ GEM
       shex (~> 0.7)
       sparql (~> 3.2)
       sparql-client (~> 3.2)
-    linkheaders-processor (0.1.10)
+    linkheaders-processor (0.1.12)
       json (~> 2.0)
       json-ld (~> 3.2)
       json-ld-preloaded (~> 3.2)

data/lib/fsp_harvester/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FspHarvester
-  VERSION = "0.1.3"
+  VERSION = "0.1.6"
 end

data/lib/fsp_harvester.rb

@@ -1,24 +1,24 @@
 # frozen_string_literal: true
 
-require_relative "fsp_harvester/version"
-require "json/ld"
-require "json/ld/preloaded"
-require "json"
-require "linkheaders/processor"
-require "addressable"
-require "tempfile"
-require "xmlsimple"
-require "nokogiri"
-require "parseconfig"
-require "rest-client"
-require "cgi"
-require "digest"
-require "open3"
-require "metainspector"
-require "rdf/xsd"
-require_relative "./metadata_object"
-require_relative "./constants"
-require_relative "./web_utils"
+require_relative 'fsp_harvester/version'
+require 'json/ld'
+require 'json/ld/preloaded'
+require 'json'
+require 'linkheaders/processor'
+require 'addressable'
+require 'tempfile'
+require 'xmlsimple'
+require 'nokogiri'
+require 'parseconfig'
+require 'rest-client'
+require 'cgi'
+require 'digest'
+require 'open3'
+require 'metainspector'
+require 'rdf/xsd'
+require_relative './metadata_object'
+require_relative './constants'
+require_relative './web_utils'
 
 module FspHarvester
   class Error < StandardError
@@ -32,28 +32,28 @@ module FspHarvester
     def self.resolve_guid(guid:)
       @meta.finalURI = [guid]
       type, url = convertToURL(guid: guid)
-      links = Array.new
-      unless type
-        @meta.warnings << ["006", guid, ""]
-        @meta.comments << "FATAL: GUID type not recognized.\n"
-      else
+      links = []
+      if type
         links, @meta = resolve_url(url: url)
+      else
+        @meta.warnings << ['006', guid, '']
+        @meta.comments << "FATAL: GUID type not recognized.\n"
       end
       [links, @meta]
     end
 
     def self.convertToURL(guid:)
       GUID_TYPES.each do |k, regex|
-        if k == "inchi" and regex.match(guid)
-          return "inchi", "https://pubchem.ncbi.nlm.nih.gov/rest/rdf/inchikey/#{guid}"
-        elsif k == "handle1" and regex.match(guid)
-          return "handle", "http://hdl.handle.net/#{guid}"
-        elsif k == "handle2" and regex.match(guid)
-          return "handle", "http://hdl.handle.net/#{guid}"
-        elsif k == "uri" and regex.match(guid)
-          return "uri", guid
-        elsif k == "doi" and regex.match(guid)
-          return "doi", "https://doi.org/#{guid}"
+        if k == 'inchi' and regex.match(guid)
+          return 'inchi', "https://pubchem.ncbi.nlm.nih.gov/rest/rdf/inchikey/#{guid}"
+        elsif k == 'handle1' and regex.match(guid)
+          return 'handle', "http://hdl.handle.net/#{guid}"
+        elsif k == 'handle2' and regex.match(guid)
+          return 'handle', "http://hdl.handle.net/#{guid}"
+        elsif k == 'uri' and regex.match(guid)
+          return 'uri', guid
+        elsif k == 'doi' and regex.match(guid)
+          return 'doi', "https://doi.org/#{guid}"
         end
       end
       [nil, nil]
@@ -66,14 +66,14 @@ module FspHarvester
       false
     end
 
-    def self.resolve_url(url:, nolinkheaders: false, header: ACCEPT_ALL_HEADER)
-      @meta.guidtype = "uri" if @meta.guidtype.nil?
+    def self.resolve_url(url:, method: :get, nolinkheaders: false, header: ACCEPT_ALL_HEADER)
+      @meta.guidtype = 'uri' if @meta.guidtype.nil?
       warn "\n\n FETCHING #{url} #{header}\n\n"
-      response = FspHarvester::WebUtils.fspfetch(url: url, headers: header)
-      warn "\n\n head #{response.headers.inspect}\n\n"
+      response = FspHarvester::WebUtils.fspfetch(url: url, headers: header, method: method)
+      warn "\n\n head #{response.headers.inspect}\n\n" if response
 
       unless response
-        @meta.warnings << ["001", url, header]
+        @meta.warnings << ['001', url, header]
         @meta.comments << "WARN: Unable to resolve #{url} using HTTP Accept header #{header}.\n"
         return [[], @meta]
       end
@@ -90,48 +90,113 @@ module FspHarvester
 
       parser = LinkHeaders::Processor.new(default_anchor: @meta.finalURI.last)
       parser.extract_and_parse(response: response)
-      factory = parser.factory  # LinkHeaders::LinkFactory
+      factory = parser.factory # LinkHeaders::LinkFactory
 
-      citeas = 0
-      describedby = 0
       warn "\n\n length #{factory.all_links.length}\n\n"
+      signpostingcheck(factory: factory)
+    end
 
+    def self.signpostingcheck(factory:)
+      citeas = 0
+      describedby = 0
       factory.all_links.each do |l|
         case l.relation
-        when "cite-as"
+        when 'cite-as'
           citeas += 1
-        when "describedby"
+        when 'item'
+          if !(l.respond_to? 'type')
+            @meta.warnings << ['011', l.href, '']
+            @meta.comments << "WARN: The resource does not follow the FAIR Signposting standard, which encourages any item links to also have a 'type' attribute.\n"
+          end
+          type = l.type if l.respond_to? 'type'
+          type = '*/*' unless type  # this becomes a frozen string
+          header = { accept: type }
+          response = FspHarvester::WebUtils.fspfetch(url: l.href, headers: header, method: :head)
+          
+          if response
+            if response.headers[:content_type] and !(type == '*/*')
+              rtype = type.gsub(%r{/}, "\/")   # because type is a frozen string
+              rtype = rtype.gsub(/\+/, '.')
+              typeregex = Regexp.new(type)
+              if response.headers[:content_type].match(typeregex)
+                warn response.headers[:content_type]
+                warn typeregex.inspect
+                @meta.comments << "INFO: item link responds according to Signposting specifications\n"
+              else
+                @meta.warnings << ['012', l.href, header]
+                @meta.comments << "WARN: Content type of returned item link does not match the 'type' attribute\n"
+              end
+            else
+              @meta.warnings << ['013', l.href, header]
+              @meta.comments << "WARN: Content type of returned item link is not specified in response headers or cannot be matched against accept headers\n"
+            end
+          else
+            @meta.warnings << ['014', l.href, header]
+            @meta.comments << "WARN: item link doesn't resolve\n"
+          end
+
+        when 'describedby'
           describedby += 1
-          unless l.respond_to? "type"
-            @meta.warnings << ["005", l.url, ""]
-            @meta.comments << "WARN: The resource does not follow the FAIR Signposting standard, which requires any describedby links to also have a 'type' attribute\n"    
+          if !(l.respond_to? 'type')
+            @meta.warnings << ['005', l.href, '']
+            @meta.comments << "WARN: The resource does not follow the FAIR Signposting standard, which requires any describedby links to also have a 'type' attribute.\n"
+          end
+          type = l.type if l.respond_to? 'type'
+          type = '*/*' unless type
+          header = { accept: type }
+          response = FspHarvester::WebUtils.fspfetch(url: l.href, headers: header, method: :head)
+          if response
+            if response.headers[:content_type] and !(type == '*/*')
+              rtype = type.gsub(%r{/}, "\/")
+              rtype = rtype.gsub(/\+/, '.')
+              typeregex = Regexp.new(rtype)
+              if response.headers[:content_type].match(typeregex)
+                warn response.headers[:content_type]
+                warn typeregex.inspect
+                @meta.comments << "INFO: describedby link responds according to Signposting specifications\n"
+              else
+                @meta.warnings << ['009', l.href, header]
+                @meta.comments << "WARN: Content type of returned describedby link does not match the 'type' attribute\n"
+              end
+            else
+              @meta.warnings << ['010', l.href, header]
+              @meta.comments << "WARN: Content type of returned describedby link is not specified in response headers or cannot be matched against accept headers\n"
+            end
+          else
+            @meta.warnings << ['008', l.href, header]
+            @meta.comments << "WARN: describedby link doesn't resolve\n"
           end
         end
       end
       if citeas > 1
-        self.check_for_conflicts(factory: factory)  # this merelty adsds to the metadata objects if there are conflicts
+        @meta.comments << "INFO: multiple cite-as links found. Checking for conflicts\n"
+        citeas = check_for_citeas_conflicts(factory: factory) # this adds to the metadata objects if there are conflicts, returns the list of unique citeas (SHOULD ONLY BE ONE!)
       end
 
       unless citeas == 1 && describedby > 0
-        @meta.warnings << ["004", "", ""]
+        @meta.warnings << ['004', '', '']
         @meta.comments << "WARN: The resource does not follow the FAIR Signposting standard, which requires exactly one cite-as header, and at least one describedby header\n"
       end
       factory.all_links
     end
 
-    def self.check_for_conflicts(factory:) # incoming: {"link1" => {"sectiontype1" => value, "sectiontype2" => value2}}
-      @meta.comments << "INFO: checking for conflicting cite-as links"
-      citeas = Array.new
+    def self.check_for_citeas_conflicts(factory:)
+      @meta.comments << 'INFO: checking for conflicting cite-as links'
+      citeas = []
       factory.all_links.each do |link|
         next unless link.relation == 'cite-as'
+
+        @meta.comments << "INFO: Adding citeas #{link.href} to the testing queue."
         citeas << link.href
       end
-      unless citeas == citeas.uniq
-        @meta.warnings << ["007", url, header]
+
+      if citeas.uniq.length == 1
+        @meta.comments << 'INFO: No conflicting cite-as links found.'
+      else  # only one allowed!
+        @meta.warnings << ['007', '', '']
         @meta.comments << "WARN: The resource does not follow the FAIR Signposting standard: Found conflicting cite-as link headers\n"
-      else
-        @meta.comments << "INFO: No conflicting cite-as links found."
       end
+      citeas.uniq
     end
   end
 end

data/lib/warnings.json

@@ -28,6 +28,47 @@
       "message": "GUID type not recognized",
       "linkout": "",
       "severity": "WARN"
+   },
+   "007": {
+      "message": "Conflicting cite-as links",
+      "linkout": "",
+      "severity": "WARN"
+   },
+   "008": {
+      "message": "describedby link does not resolve",
+      "linkout": "",
+      "severity": "WARN"
+   },
+   "009": {
+      "message": "Content-type of described-by link does not match the type attribute in the link header itself",
+      "linkout": "",
+      "severity": "WARN"
+   },
+   "010": {
+      "message": "Content-type of response from described-by link is undefined or cannot be compared to the link type",
+      "linkout": "",
+      "severity": "WARN"
+   },
+   "011": {
+      "message": "The resource does not follow the FAIR Signposting standard, which encourages any item links to have a type attribute",
+      "linkout": "",
+      "severity": "WARN"
+   },
+   "012": {
+      "message": "Content-type of response from resolving an item doesn't match the item type attribute in the link header",
+      "linkout": "",
+      "severity": "WARN"
+   },
+   "013": {
+      "message": "Content-type of response from resolving an item is undefined or cannot be compared to the link type",
+      "linkout": "",
+      "severity": "WARN"
+   },
+   "014": {
+      "message": "Item link does not resolve",
+      "linkout": "",
+      "severity": "WARN"
    }
+
    
 }

data/lib/web_utils.rb

@@ -1,13 +1,13 @@
 module FspHarvester
 
   class WebUtils
-    def self.fspfetch(url:, headers: ACCEPT_ALL_HEADER)  # we will try to retrieve turtle whenever possible
+    def self.fspfetch(url:, headers: ACCEPT_ALL_HEADER, method: :get)
       warn 'In fetch routine now.  '
 
       begin
         warn "executing call over the Web to #{url}"
         response = RestClient::Request.execute({
-                                                method: :get,
+                                                method: method,
                                                 url: url.to_s,
                                                 # user: user,
                                                 # password: pass,
@@ -26,7 +26,7 @@ module FspHarvester
         warn "EXCEPTION WITH RESPONSE! #{e.response}\n#{e.response.headers}"
         @meta.warnings << ["003", url, headers] if @meta
         @meta.comments << "WARN: HTTP error #{e} encountered when trying to resolve #{url}\n" if @meta
-        if e.response.code == 500
+        if (e.response.code == 500 or e.response.code == 404)
           return false
         else
           e.response

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fsp_harvester
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.6
 platform: ruby
 authors:
 - Mark Wilkinson
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-07-27 00:00:00.000000000 Z
+date: 2022-07-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.10
+        version: 0.1.12
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.10
+        version: 0.1.12
 - !ruby/object:Gem::Dependency
   name: metainspector
   requirement: !ruby/object:Gem::Requirement