fsp_harvester 0.1.16 → 0.1.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aec11fd57963ffb176ddb88338b9e262027c9a7d39364089ae130fb4b628bf5b
-  data.tar.gz: f8733a00de5c6c24a622235c18ba0dae208f5bac52d50607480e51fd563678c8
+  metadata.gz: 548e58dcb21d312f32780345f954f523e253c8fffaa4fbe9598032c3081adb1a
+  data.tar.gz: 7bea92aa5809b6651acb76aa2ea470c069e480783eb9d1acfd407a8a5fbd261c
 SHA512:
-  metadata.gz: c484e41aa0305f34d0bf7f82cad60b9b02106ffe80b9371c99e77b199eef9ce52818222368b8b3a3ff73d94dba89b8d7fb815d29c95ca335772946e1e9762849
-  data.tar.gz: '09dfdcc12b9176bc88c31a196893ae9ede6c35c2fd59271ca8fa5b1c29f0807ee82c8416ca5f8b7ff75c6caab71648b6d0e15d0976784cb7d34ff8686332be37'
+  metadata.gz: 5da8243a22e50244b30f5672b34e5d2f484a6dbf4125fb80a5c93cf1580dd561d6fb4c9354dcce44588f8223d5ec9eb7a8a70bd4904a67b0b59c34196194db0b
+  data.tar.gz: dfdc907ba268b0e509256000a0bd36adbcafe70a1490066acc91deef0255bc13cfe74dad3ddc31352ab6d6eff80fde6a8c3669924c9cb58febd8684a902c06a7

data/.rspec_status

@@ -1,56 +1,57 @@
 example_id                         | status | run_time        |
 ---------------------------------- | ------ | --------------- |
-./spec/cite-as_spec.rb[1:1:1]      | passed | 1.66 seconds    |
-./spec/cite-as_spec.rb[1:1:2]      | passed | 1.13 seconds    |
-./spec/cite-as_spec.rb[1:1:3]      | passed | 1.08 seconds    |
-./spec/cite-as_spec.rb[1:1:4]      | passed | 1.68 seconds    |
-./spec/cite-as_spec.rb[1:1:5]      | passed | 2.86 seconds    |
-./spec/cite-as_spec.rb[1:1:6]      | passed | 2.11 seconds    |
-./spec/cite-as_spec.rb[1:1:7]      | passed | 3.07 seconds    |
-./spec/cite-as_spec.rb[1:1:8]      | passed | 2.13 seconds    |
-./spec/cite-as_spec.rb[1:1:9]      | passed | 2.73 seconds    |
-./spec/cite-as_spec.rb[1:1:10]     | passed | 2.64 seconds    |
-./spec/cite-as_spec.rb[1:1:11]     | passed | 3.36 seconds    |
-./spec/cite-as_spec.rb[1:1:12]     | passed | 2.26 seconds    |
-./spec/cite-as_spec.rb[1:1:13]     | passed | 2.9 seconds     |
-./spec/cite-as_spec.rb[1:1:14]     | passed | 2.31 seconds    |
-./spec/cite-as_spec.rb[1:1:15]     | passed | 1.47 seconds    |
-./spec/cite-as_spec.rb[1:1:16]     | passed | 1.22 seconds    |
-./spec/cite-as_spec.rb[1:1:17]     | passed | 1.23 seconds    |
-./spec/cite-as_spec.rb[1:1:18]     | passed | 1.2 seconds     |
-./spec/cite-as_spec.rb[1:1:19]     | passed | 1.71 seconds    |
-./spec/cite-as_spec.rb[1:1:20]     | passed | 1.66 seconds    |
-./spec/cite-as_spec.rb[1:1:21]     | passed | 2.5 seconds     |
-./spec/cite-as_spec.rb[1:1:22]     | passed | 1.54 seconds    |
-./spec/cite-as_spec.rb[1:1:23]     | passed | 1.25 seconds    |
-./spec/cite-as_spec.rb[1:1:24]     | failed | 1.35 seconds    |
-./spec/cite-as_spec.rb[1:1:25]     | passed | 0.50811 seconds |
-./spec/describedby_spec.rb[1:1:1]  | passed | 3.45 seconds    |
-./spec/describedby_spec.rb[1:1:2]  | passed | 1.3 seconds     |
+./spec/cite-as_spec.rb[1:1:1]      | passed | 1.36 seconds    |
+./spec/cite-as_spec.rb[1:1:2]      | passed | 1.31 seconds    |
+./spec/cite-as_spec.rb[1:1:3]      | passed | 1.53 seconds    |
+./spec/cite-as_spec.rb[1:1:4]      | passed | 1.84 seconds    |
+./spec/cite-as_spec.rb[1:1:5]      | passed | 2.77 seconds    |
+./spec/cite-as_spec.rb[1:1:6]      | passed | 2.06 seconds    |
+./spec/cite-as_spec.rb[1:1:7]      | passed | 2.96 seconds    |
+./spec/cite-as_spec.rb[1:1:8]      | passed | 2.28 seconds    |
+./spec/cite-as_spec.rb[1:1:9]      | passed | 2.83 seconds    |
+./spec/cite-as_spec.rb[1:1:10]     | passed | 2.14 seconds    |
+./spec/cite-as_spec.rb[1:1:11]     | passed | 3.19 seconds    |
+./spec/cite-as_spec.rb[1:1:12]     | passed | 3.06 seconds    |
+./spec/cite-as_spec.rb[1:1:13]     | passed | 2.77 seconds    |
+./spec/cite-as_spec.rb[1:1:14]     | passed | 2.2 seconds     |
+./spec/cite-as_spec.rb[1:1:15]     | passed | 1.19 seconds    |
+./spec/cite-as_spec.rb[1:1:16]     | passed | 1.1 seconds     |
+./spec/cite-as_spec.rb[1:1:17]     | passed | 1.31 seconds    |
+./spec/cite-as_spec.rb[1:1:18]     | passed | 1.14 seconds    |
+./spec/cite-as_spec.rb[1:1:19]     | passed | 1.68 seconds    |
+./spec/cite-as_spec.rb[1:1:20]     | passed | 1.69 seconds    |
+./spec/cite-as_spec.rb[1:1:21]     | passed | 2.35 seconds    |
+./spec/cite-as_spec.rb[1:1:22]     | passed | 1.12 seconds    |
+./spec/cite-as_spec.rb[1:1:23]     | passed | 1.16 seconds    |
+./spec/cite-as_spec.rb[1:1:24]     | failed | 1.45 seconds    |
+./spec/cite-as_spec.rb[1:1:25]     | passed | 0.72571 seconds |
+./spec/describedby_spec.rb[1:1:1]  | passed | 3.09 seconds    |
+./spec/describedby_spec.rb[1:1:2]  | passed | 1.13 seconds    |
 ./spec/describedby_spec.rb[1:1:3]  | passed | 1.22 seconds    |
-./spec/describedby_spec.rb[1:1:4]  | passed | 1.22 seconds    |
-./spec/describedby_spec.rb[1:1:5]  | passed | 1.15 seconds    |
-./spec/describedby_spec.rb[1:1:6]  | passed | 1.04 seconds    |
-./spec/describedby_spec.rb[1:1:7]  | passed | 1.12 seconds    |
-./spec/describedby_spec.rb[1:1:8]  | passed | 2.44 seconds    |
-./spec/describedby_spec.rb[1:1:9]  | passed | 2.15 seconds    |
-./spec/describedby_spec.rb[1:1:10] | passed | 2.19 seconds    |
-./spec/describedby_spec.rb[1:1:11] | passed | 2.98 seconds    |
-./spec/describedby_spec.rb[1:1:12] | passed | 2.87 seconds    |
-./spec/describedby_spec.rb[1:1:13] | passed | 1.74 seconds    |
-./spec/describedby_spec.rb[1:1:14] | passed | 2.27 seconds    |
-./spec/describedby_spec.rb[1:1:15] | passed | 2.28 seconds    |
-./spec/fsp_harvester_spec.rb[1:1]  | passed | 0.00058 seconds |
-./spec/fsp_harvester_spec.rb[1:2]  | failed | 2.92 seconds    |
-./spec/fsp_harvester_spec.rb[1:3]  | passed | 6.87 seconds    |
-./spec/item_spec.rb[1:1:1]         | passed | 3.19 seconds    |
-./spec/item_spec.rb[1:1:2]         | passed | 2.81 seconds    |
-./spec/item_spec.rb[1:1:3]         | passed | 1.27 seconds    |
-./spec/item_spec.rb[1:1:4]         | passed | 1.76 seconds    |
-./spec/item_spec.rb[1:1:5]         | passed | 2.43 seconds    |
-./spec/item_spec.rb[1:1:6]         | passed | 2.23 seconds    |
-./spec/item_spec.rb[1:1:7]         | passed | 2.94 seconds    |
-./spec/item_spec.rb[1:1:8]         | passed | 0.52517 seconds |
-./spec/type_spec.rb[1:1:1]         | passed | 1.35 seconds    |
-./spec/type_spec.rb[1:1:2]         | passed | 1.32 seconds    |
-./spec/type_spec.rb[1:1:3]         | passed | 1.54 seconds    |
+./spec/describedby_spec.rb[1:1:4]  | passed | 1.11 seconds    |
+./spec/describedby_spec.rb[1:1:5]  | passed | 1.21 seconds    |
+./spec/describedby_spec.rb[1:1:6]  | passed | 1.24 seconds    |
+./spec/describedby_spec.rb[1:1:7]  | passed | 1.53 seconds    |
+./spec/describedby_spec.rb[1:1:8]  | passed | 2.53 seconds    |
+./spec/describedby_spec.rb[1:1:9]  | passed | 1.74 seconds    |
+./spec/describedby_spec.rb[1:1:10] | passed | 2.59 seconds    |
+./spec/describedby_spec.rb[1:1:11] | passed | 3.49 seconds    |
+./spec/describedby_spec.rb[1:1:12] | passed | 3.82 seconds    |
+./spec/describedby_spec.rb[1:1:13] | passed | 1.65 seconds    |
+./spec/describedby_spec.rb[1:1:14] | passed | 2.19 seconds    |
+./spec/describedby_spec.rb[1:1:15] | passed | 2.16 seconds    |
+./spec/fsp_harvester_spec.rb[1:1]  | passed | 0.00015 seconds |
+./spec/fsp_harvester_spec.rb[1:2]  | passed | 2.49 seconds    |
+./spec/fsp_harvester_spec.rb[1:3]  | passed | 7.06 seconds    |
+./spec/fsp_harvester_spec.rb[1:4]  | passed | 2.74 seconds    |
+./spec/item_spec.rb[1:1:1]         | passed | 3.41 seconds    |
+./spec/item_spec.rb[1:1:2]         | passed | 2.84 seconds    |
+./spec/item_spec.rb[1:1:3]         | passed | 1.15 seconds    |
+./spec/item_spec.rb[1:1:4]         | passed | 1.74 seconds    |
+./spec/item_spec.rb[1:1:5]         | passed | 2.6 seconds     |
+./spec/item_spec.rb[1:1:6]         | passed | 2.32 seconds    |
+./spec/item_spec.rb[1:1:7]         | passed | 2.81 seconds    |
+./spec/item_spec.rb[1:1:8]         | passed | 0.49717 seconds |
+./spec/type_spec.rb[1:1:1]         | passed | 1.25 seconds    |
+./spec/type_spec.rb[1:1:2]         | passed | 1.18 seconds    |
+./spec/type_spec.rb[1:1:3]         | passed | 1.58 seconds    |

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    fsp_harvester (0.1.16)
+    fsp_harvester (0.1.19)
       json (~> 2.0)
       linkeddata (~> 3.2)
-      linkheaders-processor (~> 0.1.17)
+      linkheaders-processor (~> 0.1.18)
       metainspector (~> 5.11.2)
       parseconfig (~> 1.1)
       rake (~> 13.0)
@@ -127,7 +127,7 @@ GEM
       sparql (~> 3.2, >= 3.2.4)
       sparql-client (~> 3.2, >= 3.2.1)
       yaml-ld (~> 0.0)
-    linkheaders-processor (0.1.17)
+    linkheaders-processor (0.1.18)
       json (~> 2.0)
       json-ld (~> 3.2)
       json-ld-preloaded (~> 3.2)
@@ -252,14 +252,14 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.11.0)
     rspec-support (3.11.0)
-    rubocop (1.34.1)
+    rubocop (1.35.0)
       json (~> 2.3)
       parallel (~> 1.10)
       parser (>= 3.1.2.1)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.20.0, < 2.0)
+      rubocop-ast (>= 1.20.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
     rubocop-ast (1.21.0)

data/README.md

@@ -20,6 +20,39 @@ Or install it yourself as:
 
 ## Usage
 
+```
+require 'fsp_harvester'
+
+ENV['EXTRUCT_COMMAND'] = "extruct"
+ENV['RDF_COMMAND'] = "/home/user/.rvm/gems/ruby-3.0.0/bin/rdf" # kelloggs distiller
+ENV['TIKA_COMMAND'] = "http://localhost:9998/meta" # assumes using the docker version of tika
+
+# to only follow the FAIR signposting specification:
+links, metadata = HarvesterTools::Utils.resolve_guid(guid: guid)
+
+links.each do |link|
+    puts link.href
+    puts link.relation
+end
+
+# note, you don't need to catch the return value here.  The metadata object that is passed in will be modified
+metadata = FspHarvester::Utils.gather_metadata_from_describedby_links(links: links, metadata: metadata)
+
+linkeddata = metadata.graph
+hashdata = metadata.hash
+comments = metadata.comments
+warnings = metadata.warnings
+
+# if you want to try other things like content negotiation and "scraping" from HTML, do this:
+# note, you don't need to catch the return value here.  The metadata object that is passed in will be modified
+metadata = HarvesterTools::BruteForce.begin_brute_force(guid: guid, metadata: metadata)
+
+linkeddata = metadata.graph
+hashdata = metadata.hash
+comments = metadata.comments
+warnings = metadata.warnings
+
+```
 
 
 ## Development

data/lib/constants.rb

@@ -78,31 +78,31 @@ GUID_TYPES = {
   'ark' => Regexp.new(%r{^ark:/[^\s]+$}) 
 }
 
-CONFIG = File.exist?('config.conf') ? ParseConfig.new('config.conf') : {}
-extruct = CONFIG.dig(:extruct, :command)
-extruct ||= 'extruct'
+# CONFIG = File.exist?('config.conf') ? ParseConfig.new('config.conf') : {}
+# extruct = CONFIG.dig(:extruct, :command)
+extruct = ENV['EXTRUCT_COMMAND'] || 'extruct'
 extruct.strip!
 case extruct
 when /[&|;`$\s]/
-  abort 'The Extruct command in the config file appears to be subject to command injection.  I will not continue'
+  abort 'The Extruct command appears to be subject to command injection.  I will not continue'
 when /echo/i
-  abort 'The Extruct command in the config file appears to be subject to command injection.  I will not continue'
+  abort 'The Extruct command appears to be subject to command injection.  I will not continue'
 end
 EXTRUCT_COMMAND = extruct
 
-rdf_command = CONFIG.dig(:rdf, :command)
-rdf_command ||= 'rdf'
+# rdf_command = CONFIG.dig(:rdf, :command)
+rdf_command = ENV['RDF_COMMAND'] || 'rdf'
 rdf_command.strip
 case rdf_command
 when /[&|;`$\s]/
-  abort 'The RDF command in the config file appears to be subject to command injection.  I will not continue'
+  abort 'The RDF command appears to be subject to command injection.  I will not continue'
 when /echo/i
-  abort 'The RDF command in the config file appears to be subject to command injection.  I will not continue'
+  abort 'The RDF command appears to be subject to command injection.  I will not continue'
 when !(/rdf$/ =~ $_)
   abort "this software requires that Kelloggs Distiller tool is used. The distiller command must end in 'rdf'"
 end
 RDF_COMMAND = rdf_command
 
-tika_command = CONFIG.dig(:tika, :command)
-tika_command ||= 'http://localhost:9998/meta'
+# tika_command = CONFIG.dig(:tika, :command)
+tika_command = ENV['TIKA_COMMAND'] || 'http://localhost:9998/meta'
 TIKA_COMMAND = tika_command

data/lib/fsp_harvester/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FspHarvester
-  VERSION = "0.1.16"
+  VERSION = "0.1.19"
 end

data/lib/harvester_utils.rb

@@ -4,14 +4,14 @@ module HarvesterTools
 
   class Utils
 
-    def self.resolve_guid(guid:)
-      @meta = HarvesterTools::MetadataObject.new
+    def self.resolve_guid(guid:, metadata: HarvesterTools::MetadataObject.new)
+      @meta = metadata
       @meta.all_uris = [guid]
       type, url = convertToURL(guid: guid)
       links = Array.new
       if type
         links = resolve_url(url: url, metadata: @meta)
-        @meta.links = @meta.links | links
+        @meta.links = @meta.links.append(*links)
       else
         @meta.add_warning(['006', guid, ''])
         @meta.comments << "FATAL: GUID type not recognized.\n"

data/lib/metadata_object.rb

@@ -1,15 +1,21 @@
 module HarvesterTools
   class MetadataObject
-    attr_accessor :hash, :graph, :comments, :links, :warnings, :guidtype, :full_response, :all_uris  # a hash of metadata # a RDF.rb graph of metadata  # an array of comments  # the type of GUID that was detected # will be an array of Net::HTTP::Response
+    attr_accessor :id, :hash, :graph, :comments, :links, :warnings, :guidtype, :full_response, :all_uris, :tested_guid, :score, :version, :date  # a hash of metadata # a RDF.rb graph of metadata  # an array of comments  # the type of GUID that was detected # will be an array of Net::HTTP::Response
 
-    def initialize(_params = {}) # get a name from the "new" call, or set a default
+    def initialize(id: "unidentified_metadata") # get a name from the "new" call, or set a default
+      @id = id
       @hash = {}
       @graph = RDF::Graph.new
       @comments =  []
       @warnings =  []
       @full_response = []
       @links = []
+      @guidtype = ""
       @all_uris = []
+      @tested_guid = ""
+      @score = 0
+      @version = '0.0'
+      @date = Time.now.strftime('%Y-%m-%dT%H:%M:%S.%L%z')
       w = RestClient.get("https://raw.githubusercontent.com/markwilkinson/FAIR-Signposting-Harvester/master/lib/warnings.json")
       #@warn = File.read("./lib/warnings.json")
       @warn = JSON.parse(w)

data/lib/warnings.json

@@ -115,5 +115,12 @@
                   {"Documentation": "http://dx.doi.org/10.17487/RFC8259"},
                   {"Validator": "https://jsononline.net/json-validator"}],
       "severity": "WARN"
+   },
+   "600": {
+      "message": "Data identifier cannot be unambiguously determined, therefore cannot be tested against known persistent identifier schemas",
+      "linkout": [],
+      "severity": "FAILURE"
    }
+
+
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fsp_harvester
 version: !ruby/object:Gem::Version
-  version: 0.1.16
+  version: 0.1.19
 platform: ruby
 authors:
 - Mark Wilkinson
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-08-12 00:00:00.000000000 Z
+date: 2022-08-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.17
+        version: 0.1.18
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.17
+        version: 0.1.18
 - !ruby/object:Gem::Dependency
   name: metainspector
   requirement: !ruby/object:Gem::Requirement
@@ -182,9 +182,6 @@ files:
 - bin/setup
 - example_test.rb
 - launch.json
-- lib/config.conf
-- lib/config.conf_docker
-- lib/config.conf_local
 - lib/constants.rb
 - lib/external_tools.rb
 - lib/fsp_harvester.rb
@@ -195,6 +192,9 @@ files:
 - lib/metadata_harvester.rb
 - lib/metadata_object.rb
 - lib/metadata_parser.rb
+- lib/obselete_config.conf
+- lib/obselete_config.conf_docker
+- lib/obselete_config.conf_local
 - lib/signposting_tests.rb
 - lib/warnings.json
 - lib/web_utils.rb