fsp_harvester 0.1.4 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ceb83f45b1094afb1bab24e948cf863c764cb9f045f42706017dd36c7507aa1f
-  data.tar.gz: 41fdf2bfb9166cfaff70058523f442630967721541cb9edc1fc147932f0229b3
+  metadata.gz: 7da32f3321193e93f64154c35db0c840f5e7f086451660f99b62d3f4c834e295
+  data.tar.gz: 5a0a1ff4ef6b2100accd8bab4f20d6842d25ebf88d5e600e646804da9ed24bd9
 SHA512:
-  metadata.gz: 7db52cf3b6fcfc8b4ab9a9f5b470ab97d24e192130eb54c3850a5b0c46a4dd4595748266f50322775d3078f7e372c0ba00c4a414342b1b7b82268565b230a173
-  data.tar.gz: c2f2bc421c0c0dca304d4ed7e9393b023b9d1af4fc6e0457b334296010d0e08132f44fed64202afb52df913ba6a8b69e1c91b0309b6020f8d79a3f7cb7de2a03
+  metadata.gz: dad90f81b73489a151220c132d74508832573a352b209a537bbbaf6543a90b9e132cca80ecca33d15d02eca91841642d73e795113076f4564730194b5bf1fa53
+  data.tar.gz: 5d11c2f002f4e73a4971aec0d047cd14d35f6763155d0effa9c79f419ff6fd26f853158b290d2264edd148633570f5ecd50d4e1f6c7e9d136c7d2880517bdefc

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    fsp_harvester (0.1.4)
+    fsp_harvester (0.1.7)
       json (~> 2.0)
       linkeddata (~> 3.2)
-      linkheaders-processor (~> 0.1.11)
+      linkheaders-processor (~> 0.1.13)
       metainspector (~> 5.11.2)
       parseconfig (~> 1.1)
       rake (~> 13.0)
@@ -126,7 +126,7 @@ GEM
       shex (~> 0.7)
       sparql (~> 3.2)
       sparql-client (~> 3.2)
-    linkheaders-processor (0.1.11)
+    linkheaders-processor (0.1.13)
       json (~> 2.0)
       json-ld (~> 3.2)
       json-ld-preloaded (~> 3.2)

data/Rakefile

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
 
-require "rubocop/rake_task"
+require 'rubocop/rake_task'
 
 RuboCop::RakeTask.new
 

data/lib/fsp_harvester/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FspHarvester
-  VERSION = "0.1.4"
+  VERSION = "0.1.7"  # up to date
 end

data/lib/fsp_harvester.rb

@@ -1,24 +1,25 @@
 # frozen_string_literal: true
 
-require_relative "fsp_harvester/version"
-require "json/ld"
-require "json/ld/preloaded"
-require "json"
-require "linkheaders/processor"
-require "addressable"
-require "tempfile"
-require "xmlsimple"
-require "nokogiri"
-require "parseconfig"
-require "rest-client"
-require "cgi"
-require "digest"
-require "open3"
-require "metainspector"
-require "rdf/xsd"
-require_relative "./metadata_object"
-require_relative "./constants"
-require_relative "./web_utils"
+require_relative 'fsp_harvester/version'
+require 'json/ld'
+require 'json/ld/preloaded'
+require 'json'
+require 'linkheaders/processor'
+require 'addressable'
+require 'tempfile'
+require 'xmlsimple'
+require 'nokogiri'
+require 'parseconfig'
+require 'rest-client'
+require 'cgi'
+require 'digest'
+require 'open3'
+require 'metainspector'
+require 'rdf/xsd'
+require_relative './metadata_object'
+require_relative './constants'
+require_relative './web_utils'
+require_relative './signposting_tests'
 
 module FspHarvester
   class Error < StandardError
@@ -27,33 +28,34 @@ module FspHarvester
   class Utils
     # @@distillerknown = {} # global, hash of sha256 keys of message bodies - have they been seen before t/f
     # @warnings = JSON.parse(File.read("warnings.json"))
-    @meta = FspHarvester::MetadataObject.new
+    
 
     def self.resolve_guid(guid:)
+      @meta = FspHarvester::MetadataObject.new
       @meta.finalURI = [guid]
       type, url = convertToURL(guid: guid)
       links = Array.new
-      unless type
-        @meta.warnings << ["006", guid, ""]
-        @meta.comments << "FATAL: GUID type not recognized.\n"
+      if type
+        links = resolve_url(url: url)
       else
-        links, @meta = resolve_url(url: url)
+        @meta.warnings << ['006', guid, '']
+        @meta.comments << "FATAL: GUID type not recognized.\n"
       end
       [links, @meta]
     end
 
     def self.convertToURL(guid:)
       GUID_TYPES.each do |k, regex|
-        if k == "inchi" and regex.match(guid)
-          return "inchi", "https://pubchem.ncbi.nlm.nih.gov/rest/rdf/inchikey/#{guid}"
-        elsif k == "handle1" and regex.match(guid)
-          return "handle", "http://hdl.handle.net/#{guid}"
-        elsif k == "handle2" and regex.match(guid)
-          return "handle", "http://hdl.handle.net/#{guid}"
-        elsif k == "uri" and regex.match(guid)
-          return "uri", guid
-        elsif k == "doi" and regex.match(guid)
-          return "doi", "https://doi.org/#{guid}"
+        if k == 'inchi' and regex.match(guid)
+          return 'inchi', "https://pubchem.ncbi.nlm.nih.gov/rest/rdf/inchikey/#{guid}"
+        elsif k == 'handle1' and regex.match(guid)
+          return 'handle', "http://hdl.handle.net/#{guid}"
+        elsif k == 'handle2' and regex.match(guid)
+          return 'handle', "http://hdl.handle.net/#{guid}"
+        elsif k == 'uri' and regex.match(guid)
+          return 'uri', guid
+        elsif k == 'doi' and regex.match(guid)
+          return 'doi', "https://doi.org/#{guid}"
         end
       end
       [nil, nil]
@@ -66,23 +68,23 @@ module FspHarvester
       false
     end
 
-    def self.resolve_url(url:, nolinkheaders: false, header: ACCEPT_ALL_HEADER)
-      @meta.guidtype = "uri" if @meta.guidtype.nil?
+    def self.resolve_url(url:, method: :get, nolinkheaders: false, header: ACCEPT_ALL_HEADER)
+      @meta.guidtype = 'uri' if @meta.guidtype.nil?
       warn "\n\n FETCHING #{url} #{header}\n\n"
-      response = FspHarvester::WebUtils.fspfetch(url: url, headers: header)
-      warn "\n\n head #{response.headers.inspect}\n\n"
+      response = FspHarvester::WebUtils.fspfetch(url: url, headers: header, method: method)
+      warn "\n\n head #{response.headers.inspect}\n\n" if response
 
       unless response
-        @meta.warnings << ["001", url, header]
+        @meta.warnings << ['001', url, header]
         @meta.comments << "WARN: Unable to resolve #{url} using HTTP Accept header #{header}.\n"
-        return [[], @meta]
+        return []
       end
 
       @meta.comments << "INFO: following redirection using this header led to the following URL: #{@meta.finalURI.last}.  Using the output from this URL for the next few tests..."
       @meta.full_response << response.body
 
       links = process_link_headers(response: response) unless nolinkheaders
-      [links, @meta]
+      links
     end
 
     def self.process_link_headers(response:)
@@ -90,47 +92,43 @@ module FspHarvester
 
       parser = LinkHeaders::Processor.new(default_anchor: @meta.finalURI.last)
       parser.extract_and_parse(response: response)
-      factory = parser.factory  # LinkHeaders::LinkFactory
+      factory = parser.factory # LinkHeaders::LinkFactory
 
-      citeas = 0
-      describedby = 0
-      warn "\n\n length #{factory.all_links.length}\n\n"
+      warn "\n\n length bfore #{factory.all_links.length}\n\n"
+      signpostingcheck(factory: factory)
+      warn "\n\n length aftr #{factory.all_links.length}\n\n"
+      warn "\n\n links #{factory.all_links}\n\n"
+      factory.all_links
+    end
 
+    def self.signpostingcheck(factory:)
+      citeas = Array.new
+      describedby = Array.new
+      item = Array.new
       factory.all_links.each do |l|
         case l.relation
-        when "cite-as"
-          citeas += 1
-        when "describedby"
-          describedby += 1
-          unless l.respond_to? "type"
-            @meta.warnings << ["005", l.url, ""]
-            @meta.comments << "WARN: The resource does not follow the FAIR Signposting standard, which requires any describedby links to also have a 'type' attribute\n"    
-          end
+        when 'cite-as'
+          citeas << l
+        when 'item'
+          item << l
+        when 'describedby'
+          describedby << l
         end
       end
-      if citeas > 1
-        self.check_for_conflicts(factory: factory)  # this merelty adsds to the metadata objects if there are conflicts
-      end
 
-      unless citeas == 1 && describedby > 0
-        @meta.warnings << ["004", "", ""]
-        @meta.comments << "WARN: The resource does not follow the FAIR Signposting standard, which requires exactly one cite-as header, and at least one describedby header\n"
-      end
-      factory.all_links
-    end
+      check_describedby_rules(describedby: describedby)
+      check_item_rules(item: item)
 
-    def self.check_for_conflicts(factory:) # incoming: {"link1" => {"sectiontype1" => value, "sectiontype2" => value2}}
-      @meta.comments << "INFO: checking for conflicting cite-as links"
-      citeas = Array.new
-      factory.all_links.each do |link|
-        next unless link.relation == 'cite-as'
-        citeas << link.href
+      uniqueciteas = Array.new
+      if citeas.length > 1
+        warn "INFO: multiple cite-as links found. Checking for conflicts\n"
+        @meta.comments << "INFO: multiple cite-as links found. Checking for conflicts\n"
+        uniqueciteas = check_for_citeas_conflicts(citeas: citeas) # this adds to the metadata objects if there are conflicts, returns the list of unique citeas (SHOULD ONLY BE ONE!)
       end
-      unless citeas == citeas.uniq
-        @meta.warnings << ["007", url, header]
-        @meta.comments << "WARN: The resource does not follow the FAIR Signposting standard: Found conflicting cite-as link headers\n"
-      else
-        @meta.comments << "INFO: No conflicting cite-as links found."
+
+      unless uniqueciteas == 1 && describedby.length > 0
+        @meta.warnings << ['004', '', '']
+        @meta.comments << "WARN: The resource does not follow the FAIR Signposting standard, which requires exactly one cite-as header, and at least one describedby header\n"
       end
     end
   end

data/lib/warnings.json

@@ -28,6 +28,47 @@
       "message": "GUID type not recognized",
       "linkout": "",
       "severity": "WARN"
+   },
+   "007": {
+      "message": "Conflicting cite-as links",
+      "linkout": "",
+      "severity": "WARN"
+   },
+   "008": {
+      "message": "describedby link does not resolve",
+      "linkout": "",
+      "severity": "WARN"
+   },
+   "009": {
+      "message": "Content-type of described-by link does not match the type attribute in the link header itself",
+      "linkout": "",
+      "severity": "WARN"
+   },
+   "010": {
+      "message": "Content-type of response from described-by link is undefined or cannot be compared to the link type",
+      "linkout": "",
+      "severity": "WARN"
+   },
+   "011": {
+      "message": "The resource does not follow the FAIR Signposting standard, which encourages any item links to have a type attribute",
+      "linkout": "",
+      "severity": "WARN"
+   },
+   "012": {
+      "message": "Content-type of response from resolving an item doesn't match the item type attribute in the link header",
+      "linkout": "",
+      "severity": "WARN"
+   },
+   "013": {
+      "message": "Content-type of response from resolving an item is undefined or cannot be compared to the link type",
+      "linkout": "",
+      "severity": "WARN"
+   },
+   "014": {
+      "message": "Item link does not resolve",
+      "linkout": "",
+      "severity": "WARN"
    }
+
    
 }

data/lib/web_utils.rb

@@ -1,13 +1,13 @@
 module FspHarvester
 
   class WebUtils
-    def self.fspfetch(url:, headers: ACCEPT_ALL_HEADER)  # we will try to retrieve turtle whenever possible
+    def self.fspfetch(url:, headers: ACCEPT_ALL_HEADER, method: :get)
       warn 'In fetch routine now.  '
 
       begin
         warn "executing call over the Web to #{url}"
         response = RestClient::Request.execute({
-                                                method: :get,
+                                                method: method,
                                                 url: url.to_s,
                                                 # user: user,
                                                 # password: pass,
@@ -26,7 +26,7 @@ module FspHarvester
         warn "EXCEPTION WITH RESPONSE! #{e.response}\n#{e.response.headers}"
         @meta.warnings << ["003", url, headers] if @meta
         @meta.comments << "WARN: HTTP error #{e} encountered when trying to resolve #{url}\n" if @meta
-        if e.response.code == 500
+        if (e.response.code == 500 or e.response.code == 404)
           return false
         else
           e.response

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fsp_harvester
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.7
 platform: ruby
 authors:
 - Mark Wilkinson
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-07-27 00:00:00.000000000 Z
+date: 2022-08-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.11
+        version: 0.1.13
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.11
+        version: 0.1.13
 - !ruby/object:Gem::Dependency
   name: metainspector
   requirement: !ruby/object:Gem::Requirement