fsp_harvester 0.1.9 → 0.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 923ccb362ef4a71fa2f221b1b224dbd5d3ec78a14cc9da0e12a1e0df804162ff
-  data.tar.gz: e70a9a994c504a0867ab10e05c07d440a76677d7ff27e27b9bdb0c3338c02ffe
+  metadata.gz: 96405252194bfb8df419e61c17f113648e8bd254c034577107d8be161ed4753d
+  data.tar.gz: 62117a821bcea28d3ee06ccccf085a3b3150b7f33e1b67594c1d21b5ce23d345
 SHA512:
-  metadata.gz: d819cb1c19d40f8a093b723cbeb4273d122d3207df2d874558327fa6a2622be1bf2c671cc9dfebee74c689722825c2dd1957f8be6bfcf2c14c09097c1dc05a5b
-  data.tar.gz: 508a484d93eab373d0389c11f4361068d5586759a73573901ea54f4d2a028f713f71bb492976785499f1074eb8b359d79be550c09b196eb82838a1e401398fc4
+  metadata.gz: d30011cfd562b090d6a08895d0ac004d8f9c4aafbe0dc4ff75be282f896c8ae6b83b5fd826db731fc248486657d66df89fd2500df9f38c1045d6fa201296a46b
+  data.tar.gz: 3064bf42a8a83ac6a89cae8635eeebcb71d22d1a923672f8379e7122e97eac527af46a710a4714b235a3671c93ea1142b2aa44bb907aa1a072918e43c31373e0

data/.rspec_status

@@ -1,55 +1,55 @@
 example_id                         | status | run_time        |
 ---------------------------------- | ------ | --------------- |
-./spec/cite-as_spec.rb[1:1:1]      | passed | 1.17 seconds    |
-./spec/cite-as_spec.rb[1:1:2]      | passed | 0.98776 seconds |
-./spec/cite-as_spec.rb[1:1:3]      | passed | 0.69753 seconds |
-./spec/cite-as_spec.rb[1:1:4]      | passed | 1.31 seconds    |
-./spec/cite-as_spec.rb[1:1:5]      | passed | 2.07 seconds    |
-./spec/cite-as_spec.rb[1:1:6]      | passed | 1.45 seconds    |
-./spec/cite-as_spec.rb[1:1:7]      | passed | 2.75 seconds    |
-./spec/cite-as_spec.rb[1:1:8]      | passed | 1.83 seconds    |
-./spec/cite-as_spec.rb[1:1:9]      | passed | 2.51 seconds    |
-./spec/cite-as_spec.rb[1:1:10]     | passed | 1.73 seconds    |
-./spec/cite-as_spec.rb[1:1:11]     | passed | 2.35 seconds    |
-./spec/cite-as_spec.rb[1:1:12]     | passed | 2.01 seconds    |
-./spec/cite-as_spec.rb[1:1:13]     | passed | 2.56 seconds    |
+./spec/cite-as_spec.rb[1:1:1]      | passed | 1.05 seconds    |
+./spec/cite-as_spec.rb[1:1:2]      | passed | 0.98315 seconds |
+./spec/cite-as_spec.rb[1:1:3]      | passed | 0.67905 seconds |
+./spec/cite-as_spec.rb[1:1:4]      | passed | 1.3 seconds     |
+./spec/cite-as_spec.rb[1:1:5]      | passed | 2.13 seconds    |
+./spec/cite-as_spec.rb[1:1:6]      | passed | 1.47 seconds    |
+./spec/cite-as_spec.rb[1:1:7]      | passed | 2.35 seconds    |
+./spec/cite-as_spec.rb[1:1:8]      | passed | 1.69 seconds    |
+./spec/cite-as_spec.rb[1:1:9]      | passed | 2.34 seconds    |
+./spec/cite-as_spec.rb[1:1:10]     | passed | 1.66 seconds    |
+./spec/cite-as_spec.rb[1:1:11]     | passed | 2.36 seconds    |
+./spec/cite-as_spec.rb[1:1:12]     | passed | 1.65 seconds    |
+./spec/cite-as_spec.rb[1:1:13]     | passed | 2.37 seconds    |
 ./spec/cite-as_spec.rb[1:1:14]     | passed | 1.68 seconds    |
-./spec/cite-as_spec.rb[1:1:15]     | passed | 1.06 seconds    |
-./spec/cite-as_spec.rb[1:1:16]     | passed | 1.03 seconds    |
-./spec/cite-as_spec.rb[1:1:17]     | passed | 0.94321 seconds |
-./spec/cite-as_spec.rb[1:1:18]     | passed | 1.1 seconds     |
-./spec/cite-as_spec.rb[1:1:19]     | passed | 1.45 seconds    |
-./spec/cite-as_spec.rb[1:1:20]     | passed | 1.53 seconds    |
-./spec/cite-as_spec.rb[1:1:21]     | passed | 1.64 seconds    |
-./spec/cite-as_spec.rb[1:1:22]     | passed | 1.01 seconds    |
-./spec/cite-as_spec.rb[1:1:23]     | passed | 1.09 seconds    |
-./spec/cite-as_spec.rb[1:1:24]     | failed | 1.22 seconds    |
-./spec/cite-as_spec.rb[1:1:25]     | passed | 0.38248 seconds |
-./spec/describedby_spec.rb[1:1:1]  | passed | 2.24 seconds    |
-./spec/describedby_spec.rb[1:1:2]  | passed | 1.08 seconds    |
-./spec/describedby_spec.rb[1:1:3]  | passed | 1 second        |
-./spec/describedby_spec.rb[1:1:4]  | passed | 1.14 seconds    |
-./spec/describedby_spec.rb[1:1:5]  | passed | 1.03 seconds    |
-./spec/describedby_spec.rb[1:1:6]  | passed | 0.81364 seconds |
-./spec/describedby_spec.rb[1:1:7]  | passed | 0.77543 seconds |
-./spec/describedby_spec.rb[1:1:8]  | passed | 2.01 seconds    |
-./spec/describedby_spec.rb[1:1:9]  | passed | 1.35 seconds    |
-./spec/describedby_spec.rb[1:1:10] | passed | 1.73 seconds    |
-./spec/describedby_spec.rb[1:1:11] | passed | 2.36 seconds    |
-./spec/describedby_spec.rb[1:1:12] | passed | 2.73 seconds    |
-./spec/describedby_spec.rb[1:1:13] | passed | 1.5 seconds     |
-./spec/describedby_spec.rb[1:1:14] | passed | 1.8 seconds     |
-./spec/describedby_spec.rb[1:1:15] | passed | 1.65 seconds    |
-./spec/fsp_harvester_spec.rb[1:1]  | passed | 0.00053 seconds |
-./spec/fsp_harvester_spec.rb[1:2]  | passed | 1.76 seconds    |
-./spec/item_spec.rb[1:1:1]         | passed | 2.08 seconds    |
-./spec/item_spec.rb[1:1:2]         | passed | 2.27 seconds    |
-./spec/item_spec.rb[1:1:3]         | passed | 1.22 seconds    |
-./spec/item_spec.rb[1:1:4]         | passed | 1.61 seconds    |
-./spec/item_spec.rb[1:1:5]         | passed | 1.74 seconds    |
-./spec/item_spec.rb[1:1:6]         | passed | 1.95 seconds    |
-./spec/item_spec.rb[1:1:7]         | passed | 3.59 seconds    |
-./spec/item_spec.rb[1:1:8]         | passed | 0.41001 seconds |
-./spec/type_spec.rb[1:1:1]         | passed | 1.14 seconds    |
-./spec/type_spec.rb[1:1:2]         | passed | 0.94799 seconds |
-./spec/type_spec.rb[1:1:3]         | passed | 1.04 seconds    |
+./spec/cite-as_spec.rb[1:1:15]     | passed | 1.01 seconds    |
+./spec/cite-as_spec.rb[1:1:16]     | passed | 1.02 seconds    |
+./spec/cite-as_spec.rb[1:1:17]     | passed | 1.04 seconds    |
+./spec/cite-as_spec.rb[1:1:18]     | passed | 0.98558 seconds |
+./spec/cite-as_spec.rb[1:1:19]     | passed | 1.36 seconds    |
+./spec/cite-as_spec.rb[1:1:20]     | passed | 1.37 seconds    |
+./spec/cite-as_spec.rb[1:1:21]     | passed | 1.71 seconds    |
+./spec/cite-as_spec.rb[1:1:22]     | passed | 0.93618 seconds |
+./spec/cite-as_spec.rb[1:1:23]     | passed | 0.94888 seconds |
+./spec/cite-as_spec.rb[1:1:24]     | failed | 1.03 seconds    |
+./spec/cite-as_spec.rb[1:1:25]     | passed | 0.3531 seconds  |
+./spec/describedby_spec.rb[1:1:1]  | passed | 2.12 seconds    |
+./spec/describedby_spec.rb[1:1:2]  | passed | 0.96254 seconds |
+./spec/describedby_spec.rb[1:1:3]  | passed | 0.92669 seconds |
+./spec/describedby_spec.rb[1:1:4]  | passed | 0.92801 seconds |
+./spec/describedby_spec.rb[1:1:5]  | passed | 1 second        |
+./spec/describedby_spec.rb[1:1:6]  | passed | 0.66763 seconds |
+./spec/describedby_spec.rb[1:1:7]  | passed | 0.66021 seconds |
+./spec/describedby_spec.rb[1:1:8]  | passed | 1.89 seconds    |
+./spec/describedby_spec.rb[1:1:9]  | passed | 1.3 seconds     |
+./spec/describedby_spec.rb[1:1:10] | passed | 1.7 seconds     |
+./spec/describedby_spec.rb[1:1:11] | passed | 2.28 seconds    |
+./spec/describedby_spec.rb[1:1:12] | passed | 2.27 seconds    |
+./spec/describedby_spec.rb[1:1:13] | passed | 1.39 seconds    |
+./spec/describedby_spec.rb[1:1:14] | passed | 1.65 seconds    |
+./spec/describedby_spec.rb[1:1:15] | passed | 1.7 seconds     |
+./spec/fsp_harvester_spec.rb[1:1]  | passed | 0.00215 seconds |
+./spec/fsp_harvester_spec.rb[1:2]  | failed | 0.00021 seconds |
+./spec/item_spec.rb[1:1:1]         | passed | 2.04 seconds    |
+./spec/item_spec.rb[1:1:2]         | passed | 2 seconds       |
+./spec/item_spec.rb[1:1:3]         | passed | 0.92924 seconds |
+./spec/item_spec.rb[1:1:4]         | passed | 1.36 seconds    |
+./spec/item_spec.rb[1:1:5]         | passed | 1.71 seconds    |
+./spec/item_spec.rb[1:1:6]         | passed | 1.68 seconds    |
+./spec/item_spec.rb[1:1:7]         | passed | 2.37 seconds    |
+./spec/item_spec.rb[1:1:8]         | passed | 0.34241 seconds |
+./spec/type_spec.rb[1:1:1]         | passed | 0.9855 seconds  |
+./spec/type_spec.rb[1:1:2]         | passed | 0.96202 seconds |
+./spec/type_spec.rb[1:1:3]         | passed | 0.96005 seconds |

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    fsp_harvester (0.1.9)
+    fsp_harvester (0.1.10)
       json (~> 2.0)
       linkeddata (~> 3.2)
-      linkheaders-processor (~> 0.1.15)
+      linkheaders-processor (~> 0.1.16)
       metainspector (~> 5.11.2)
       parseconfig (~> 1.1)
       rake (~> 13.0)
@@ -126,7 +126,7 @@ GEM
       shex (~> 0.7)
       sparql (~> 3.2)
       sparql-client (~> 3.2)
-    linkheaders-processor (0.1.15)
+    linkheaders-processor (0.1.16)
       json (~> 2.0)
       json-ld (~> 3.2)
       json-ld-preloaded (~> 3.2)

data/lib/fsp_harvester/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FspHarvester
-  VERSION = "0.1.9"
+  VERSION = "0.1.10"
 end

data/lib/fsp_harvester.rb

@@ -42,7 +42,7 @@ module FspHarvester
         links = resolve_url(url: url)
         @meta.links << links
       else
-        @meta.warnings << ['006', guid, '']
+        @meta.add_warning(['006', guid, ''])
         @meta.comments << "FATAL: GUID type not recognized.\n"
       end
       [links, @meta]
@@ -89,7 +89,7 @@ module FspHarvester
       warn "\n\n head #{response.headers.inspect}\n\n" if response
 
       unless response
-        @meta.warnings << ['001', url, header]
+        @meta.add_warning(['001', url, header])
         @meta.comments << "WARN: Unable to resolve #{url} using HTTP Accept header #{header}.\n"
         return []
       end
@@ -144,12 +144,12 @@ module FspHarvester
       end
 
       unless citeas.length == 1 && describedby.length > 0
-        @meta.warnings << ['004', '', '']
+        @meta.add_warning(['004', '', ''])
         @meta.comments << "WARN: The resource does not follow the FAIR Signposting standard, which requires exactly one cite-as header, and at least one describedby header\n"
       end
 
       unless types.length >=1
-        @meta.warnings << ['015', '', '']
+        @meta.add_warning(['015', '', ''])
         @meta.comments << "WARN: The resource does not follow the FAIR Signposting standard, which requires one or two 'type' link headers\n"
       end
     end

data/lib/fsp_metadata_external_tools.rb

@@ -40,7 +40,7 @@ module FspHarvester
         warn "DIST RESULT: #{result}"
         if result !~ /@context/i # failure returns nil
           @meta.comments << "WARN: The Distiller tool failed to find parseable data in the body, perhaps due to incorrectly formatted HTML..\n"
-          @meta.warnings << ['018', '', '']
+          @meta.add_warning(['018', '', ''])
         else
           @meta.comments << "INFO: The Distiller found parseable data.  Parsing as JSON-LD\n"
           parse_rdf(result: result, content_type: "application/ld+json")
@@ -58,10 +58,10 @@ module FspHarvester
 
       if result.to_s.match(/(Failed\sto\sextract.*?)\n/)
         @meta.comments << "WARN: extruct threw an error #{Regexp.last_match(1)} when attempting to parse return value (message body) of #{uri}.\n"
-        @meta.warnings << ['019', '', '']
+        @meta.add_warning(['019', '', ''])
         if result.to_s.match(/(ValueError:.*?)\n/)
           @meta.comments << "WARN: extruct error was #{Regexp.last_match(1)}\n"
-          @meta.warnings << ['019', '', '']
+          @meta.add_warning(['019', '', ''])
         end
       elsif result.to_s.match(/^\s+?\{/) or result.to_s.match(/^\s+\[/) # this is JSON
         json = JSON.parse result

data/lib/fsp_metadata_harvester.rb

@@ -21,7 +21,7 @@ module FspHarvester
 
         abbreviation, content_type = attempt_to_detect_type(body: response.body, headers: response.headers)
         unless abbreviation
-          @meta.warnings << ['017', url, header]
+          @meta.add_warning(['017', url, header])
           @meta.comments << "WARN: metadata format returned from #{url} using Accept header #{header} is not recognized.  Processing will end now.\n"
           next
         end
@@ -56,7 +56,7 @@ module FspHarvester
       url = link.href
       response = FspHarvester::WebUtils.fspfetch(url: url, method: :get, headers: header)
       unless response
-        @meta.warnings << ['016', url, header]
+        @meta.add_warning(['016', url, header])
         @meta.comments << "WARN: Unable to resolve describedby link #{url} using HTTP Accept header #{header}.\n"
       end
       response
@@ -87,7 +87,7 @@ module FspHarvester
       end
 
       unless content_type
-        @meta.warnings << ['017', url, header]
+        @meta.add_warning(['017', url, header])
         @meta.comments << "WARN: metadata format returned from #{url} using Accept header #{header} is not recognized.  Processing will end now.\n"
       end
       [abbreviation, content_type]

data/lib/fsp_metadata_parser.rb

@@ -24,7 +24,7 @@ module FspHarvester
         hash = XmlSimple.xml_in(body)
       rescue
         @meta.comments << "CRITICAL: Malformed XML detected.  Cannot process metadata.\n"
-        @meta.warnings << ['020', '', '']
+        @meta.add_warning(['020', '', ''])
       end
       @meta.comments << "INFO: The XML is being merged in the metadata object\n"
       @meta.hash.merge hash
@@ -35,7 +35,7 @@ module FspHarvester
         hash = JSON.parse(body)
       rescue
         @meta.comments << "CRITICAL: Malformed JSON detected.  Cannot process metadata.\n"
-        @meta.warnings << ['021', '', '']
+        @meta.add_warning(['021', '', ''])
       end
       @meta.comments << "INFO: The JSON is being merged in the metadata object\n"
       @meta.hash.merge hash
@@ -48,20 +48,20 @@ module FspHarvester
     def parse_rdf(body:, content_type:)
       unless body
         @meta.comments << "CRITICAL: The response message body component appears to have no content.\n"
-        @meta.warnings << ['018', '', '']
+        @meta.add_warning(['018', '', ''])
         return
       end
 
       unless body.match(/\w/)
         @meta.comments << "CRITICAL: The response message body component appears to have no content.\n"
-        @meta.warnings << ['018', '', '']
+        @meta.add_warning(['018', '', ''])
         return
       end
 
       rdfformat = RDF::Format.for(content_type: content_type)
       unless rdfformat
         @meta.comments << "CRITICAL: Found what appears to be RDF (sample:  #{body[0..300].delete!("\n")}), but it could not find a parser.  Please report this error, along with the GUID of the resource, to the maintainer of the system.\n"
-        @meta.warnings << ['018', '', '']
+        @meta.add_warning(['018', '', ''])
         return
       end
 
@@ -77,7 +77,7 @@ module FspHarvester
           reader = rdfformat.reader.new(body)
         rescue Exception => e
           @meta.comments << "WARN: Though linked data was found, it failed to parse (Exception #{e}).  This likely indicates some syntax error in the data.  As a result, no metadata will be extracted from this message.\n"
-          @meta.warnings << ['018', '', '']
+          @meta.add_warning(['018', '', ''])
           return
         end
 
@@ -97,11 +97,11 @@ module FspHarvester
         rescue RDF::ReaderError => e
           @meta.comments << "CRITICAL: The Linked Data was malformed and caused the parser to crash with error message: #{e.message} ||  (sample of what was parsed:  #{body[0..300].delete("\n")})\n"
           warn "CRITICAL: The Linked Data was malformed and caused the parser to crash with error message: #{e.message} ||  (sample of what was parsed:  #{body[0..300].delete("\n")})\n"
-          @meta.warnings << ['018', '', '']
+          @meta.add_warning(['018', '', ''])
         rescue Exception => e
           meta.comments << "CRITICAL: An unknown error occurred while parsing the (apparent) Linked Data (sample of what was parsed:  #{body[0..300].delete("\n")}).  Moving on...\n"
           warn "\n\nCRITICAL: #{e.inspect} An unknown error occurred while parsing the (apparent) Linked Data (full body:  #{body}).  Moving on...\n"
-          @meta.warnings << ['018', '', '']
+          @meta.add_warning(['018', '', ''])
         end
       end
     end

data/lib/metadata_object.rb

@@ -10,6 +10,8 @@ module FspHarvester
       @full_response = []
       @links = []
       @all_uris = []
+      @warn = File.read("./lib/warnings.json")
+      @warn = JSON.parse(@warn)
     end
 
     def merge_hash(hash)
@@ -25,6 +27,16 @@ module FspHarvester
     def rdf
       graph
     end
+
+    def add_warning(warning)
+      id = warning[0]
+      url = warning[1]
+      headers = warning[2]
+      message = @warn[id]['message']
+      linkout = @warn[id]['linkout']
+      severity = @warn[id]['severity']
+      self.warnings << {"id" => id, "message" => message, "severity" => severity, "linkout" => linkout, "processed_url" => url, "accept_headers": headers}
+    end
   end
 
   class Cache

data/lib/signposting_tests.rb

@@ -9,7 +9,7 @@ def check_for_citeas_conflicts(citeas: )
 
   if citeas_hrefs.length > 1
     @meta.comments << 'INFO: Found multiple non-identical cite-as links.'
-    @meta.warnings << ['007', '', '']
+    @meta.add_warning(['007', '', ''])
     @meta.comments << "WARN: The resource does not follow the FAIR Signposting standard: Found conflicting cite-as link headers.\n"
   end
   citeas_hrefs.values  # return list of unique links
@@ -19,7 +19,7 @@ end
 def check_describedby_rules(describedby:)
   describedby.each do |l|
     unless l.respond_to? 'type'
-      @meta.warnings << ['005', l.href, '']
+      @meta.add_warning(['005', l.href, ''])
       @meta.comments << "WARN: The resource does not follow the FAIR Signposting standard, which requires any describedby links to also have a 'type' attribute.\n"
     end
     type = l.type if l.respond_to? 'type'
@@ -37,15 +37,15 @@ def check_describedby_rules(describedby:)
         if responsetype == type
           @meta.comments << "INFO: describedby link responds according to Signposting specifications\n"
         else
-          @meta.warnings << ['009', l.href, header]
+          @meta.add_warning(['009', l.href, header])
           @meta.comments << "WARN: Content type of returned describedby link #{responsetype}does not match the 'type' attribute #{type}\n"
         end
       else
-        @meta.warnings << ['010', l.href, header]
+        @meta.add_warning(['010', l.href, header])
         @meta.comments << "WARN: Content type of returned describedby link is not specified in response headers or cannot be matched against accept headers\n"
       end
     else
-      @meta.warnings << ['008', l.href, header]
+      @meta.add_warning(['008', l.href, header])
       @meta.comments << "WARN: describedby link doesn't resolve\n"
     end
   end
@@ -54,7 +54,7 @@ end
 def check_item_rules(item:)
   item.each do |l| # l = LinkHeaders::Link
     unless l.respond_to? 'type'
-      @meta.warnings << ['011', l.href, '']
+      @meta.add_warning(['011', l.href, ''])
       @meta.comments << "WARN: The resource does not follow the FAIR Signposting standard, which encourages any item links to also have a 'type' attribute.\n"
     end
     type = l.type if l.respond_to? 'type'
@@ -72,15 +72,15 @@ def check_item_rules(item:)
           warn typeregex.inspect
           @meta.comments << "INFO: item link responds according to Signposting specifications\n"
         else
-          @meta.warnings << ['012', l.href, header]
+          @meta.add_warning(['012', l.href, header])
           @meta.comments << "WARN: Content type of returned item link does not match the 'type' attribute\n"
         end
       else
-        @meta.warnings << ['013', l.href, header]
+        @meta.add_warning(['013', l.href, header])
         @meta.comments << "WARN: Content type of returned item link is not specified in response headers or cannot be matched against accept headers\n"
       end
     else
-      @meta.warnings << ['014', l.href, header]
+      @meta.add_warning(['014', l.href, header])
       @meta.comments << "WARN: item link doesn't resolve\n"
     end
   end

data/lib/warnings.json

@@ -103,7 +103,7 @@
       "message": "JSON parsing error - unable to process JSON document.",
       "linkout": "",
       "severity": "WARN"
-   },
+   }
 
 
    

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fsp_harvester
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.1.10
 platform: ruby
 authors:
 - Mark Wilkinson
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-08-05 00:00:00.000000000 Z
+date: 2022-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json