fsp_harvester 0.1.15 → 0.1.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9e0ffd5048e360ce8e8cced890a586664af797065d2c8d6312927d694835e84b
-  data.tar.gz: 840269a8b28da70bed8c5e46674ff3730cbee66f624064cab84f98d5b3a2ff00
+  metadata.gz: 3f6bed703ae1a03ff30a1abb88e54b033a6ffbd24df5cacc70b2e0662af7e1be
+  data.tar.gz: 49370b82123eb0b7b6c92fd603996cec6909becf35fc789885de218aae0fb446
 SHA512:
-  metadata.gz: 4c01cc88a8f57e024c7aeed89a8251d97b130bca987dc14d914e87fa87ea744d3de7ab11ca340b0456f295edafdd872d4f63d0f0ef23dbe9c3cc8ebc97a64ae5
-  data.tar.gz: 2c274758ec874bb1c25ebd5286ecbc2b7e91205430a94cf3ada9c7350511fe362532f2c6d213a6fd6657ccdf7184df9c0eaf9c2461c0d25dc87da00b2aded390
+  metadata.gz: 2abbbfba153e08b83e832640942c978f9612437d1ee7fc3891122be96bfd8da442c2460d8a7b6f303ecc469b6de568907ed3e592a359513d8ab23966fad39786
+  data.tar.gz: 146004aae9a8495523b2c2578a84a8268b1398f005fdd98788b816f256c98128c34a537e68c078fa681e262bec3a2d0890e9226168b85735c95742f88848a0d0

data/.rspec_status

@@ -25,24 +25,25 @@ example_id                         | status | run_time        |
 ./spec/cite-as_spec.rb[1:1:23]     | passed | 1.25 seconds    |
 ./spec/cite-as_spec.rb[1:1:24]     | failed | 1.35 seconds    |
 ./spec/cite-as_spec.rb[1:1:25]     | passed | 0.50811 seconds |
-./spec/describedby_spec.rb[1:1:1]  | passed | 3.45 seconds    |
-./spec/describedby_spec.rb[1:1:2]  | passed | 1.3 seconds     |
-./spec/describedby_spec.rb[1:1:3]  | passed | 1.22 seconds    |
-./spec/describedby_spec.rb[1:1:4]  | passed | 1.22 seconds    |
-./spec/describedby_spec.rb[1:1:5]  | passed | 1.15 seconds    |
-./spec/describedby_spec.rb[1:1:6]  | passed | 1.04 seconds    |
-./spec/describedby_spec.rb[1:1:7]  | passed | 1.12 seconds    |
-./spec/describedby_spec.rb[1:1:8]  | passed | 2.44 seconds    |
-./spec/describedby_spec.rb[1:1:9]  | passed | 2.15 seconds    |
-./spec/describedby_spec.rb[1:1:10] | passed | 2.19 seconds    |
-./spec/describedby_spec.rb[1:1:11] | passed | 2.98 seconds    |
-./spec/describedby_spec.rb[1:1:12] | passed | 2.87 seconds    |
-./spec/describedby_spec.rb[1:1:13] | passed | 1.74 seconds    |
-./spec/describedby_spec.rb[1:1:14] | passed | 2.27 seconds    |
-./spec/describedby_spec.rb[1:1:15] | passed | 2.28 seconds    |
-./spec/fsp_harvester_spec.rb[1:1]  | passed | 0.00058 seconds |
-./spec/fsp_harvester_spec.rb[1:2]  | failed | 2.92 seconds    |
-./spec/fsp_harvester_spec.rb[1:3]  | passed | 6.87 seconds    |
+./spec/describedby_spec.rb[1:1:1]  | passed | 4.39 seconds    |
+./spec/describedby_spec.rb[1:1:2]  | passed | 1.63 seconds    |
+./spec/describedby_spec.rb[1:1:3]  | passed | 1.49 seconds    |
+./spec/describedby_spec.rb[1:1:4]  | passed | 1.21 seconds    |
+./spec/describedby_spec.rb[1:1:5]  | passed | 1.17 seconds    |
+./spec/describedby_spec.rb[1:1:6]  | passed | 1.11 seconds    |
+./spec/describedby_spec.rb[1:1:7]  | passed | 1.11 seconds    |
+./spec/describedby_spec.rb[1:1:8]  | passed | 2.34 seconds    |
+./spec/describedby_spec.rb[1:1:9]  | passed | 2.09 seconds    |
+./spec/describedby_spec.rb[1:1:10] | passed | 2.69 seconds    |
+./spec/describedby_spec.rb[1:1:11] | passed | 3.02 seconds    |
+./spec/describedby_spec.rb[1:1:12] | passed | 2.9 seconds     |
+./spec/describedby_spec.rb[1:1:13] | passed | 2.27 seconds    |
+./spec/describedby_spec.rb[1:1:14] | passed | 2.53 seconds    |
+./spec/describedby_spec.rb[1:1:15] | passed | 2.33 seconds    |
+./spec/fsp_harvester_spec.rb[1:1]  | passed | 0.00101 seconds |
+./spec/fsp_harvester_spec.rb[1:2]  | passed | 3.15 seconds    |
+./spec/fsp_harvester_spec.rb[1:3]  | passed | 7.1 seconds     |
+./spec/fsp_harvester_spec.rb[1:4]  | passed | 2.47 seconds    |
 ./spec/item_spec.rb[1:1:1]         | passed | 3.19 seconds    |
 ./spec/item_spec.rb[1:1:2]         | passed | 2.81 seconds    |
 ./spec/item_spec.rb[1:1:3]         | passed | 1.27 seconds    |

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fsp_harvester (0.1.15)
+    fsp_harvester (0.1.18)
       json (~> 2.0)
       linkeddata (~> 3.2)
       linkheaders-processor (~> 0.1.17)

data/README.md

@@ -20,6 +20,39 @@ Or install it yourself as:
 
 ## Usage
 
+```
+require 'fsp_harvester'
+
+ENV['EXTRUCT_COMMAND'] = "extruct"
+ENV['RDF_COMMAND'] = "/home/user/.rvm/gems/ruby-3.0.0/bin/rdf" # kelloggs distiller
+ENV['TIKA_COMMAND'] = "http://localhost:9998/meta" # assumes using the docker version of tika
+
+# to only follow the FAIR signposting specification:
+links, metadata = HarvesterTools::Utils.resolve_guid(guid: guid)
+
+links.each do |link|
+    puts link.href
+    puts link.relation
+end
+
+# note, you don't need to catch the return value here.  The metadata object that is passed in will be modified
+metadata = FspHarvester::Utils.gather_metadata_from_describedby_links(links: links, metadata: metadata)
+
+linkeddata = metadata.graph
+hashdata = metadata.hash
+comments = metadata.comments
+warnings = metadata.warnings
+
+# if you want to try other things like content negotiation and "scraping" from HTML, do this:
+# note, you don't need to catch the return value here.  The metadata object that is passed in will be modified
+metadata = HarvesterTools::BruteForce.begin_brute_force(guid: guid, metadata: metadata)
+
+linkeddata = metadata.graph
+hashdata = metadata.hash
+comments = metadata.comments
+warnings = metadata.warnings
+
+```
 
 
 ## Development

data/lib/constants.rb

@@ -78,31 +78,31 @@ GUID_TYPES = {
   'ark' => Regexp.new(%r{^ark:/[^\s]+$}) 
 }
 
-CONFIG = File.exist?('config.conf') ? ParseConfig.new('config.conf') : {}
-extruct = CONFIG.dig(:extruct, :command)
-extruct ||= 'extruct'
+# CONFIG = File.exist?('config.conf') ? ParseConfig.new('config.conf') : {}
+# extruct = CONFIG.dig(:extruct, :command)
+extruct = ENV['EXTRUCT_COMMAND'] || 'extruct'
 extruct.strip!
 case extruct
 when /[&|;`$\s]/
-  abort 'The Extruct command in the config file appears to be subject to command injection.  I will not continue'
+  abort 'The Extruct command appears to be subject to command injection.  I will not continue'
 when /echo/i
-  abort 'The Extruct command in the config file appears to be subject to command injection.  I will not continue'
+  abort 'The Extruct command appears to be subject to command injection.  I will not continue'
 end
 EXTRUCT_COMMAND = extruct
 
-rdf_command = CONFIG.dig(:rdf, :command)
-rdf_command ||= 'rdf'
+# rdf_command = CONFIG.dig(:rdf, :command)
+rdf_command = ENV['RDF_COMMAND'] || 'rdf'
 rdf_command.strip
 case rdf_command
 when /[&|;`$\s]/
-  abort 'The RDF command in the config file appears to be subject to command injection.  I will not continue'
+  abort 'The RDF command appears to be subject to command injection.  I will not continue'
 when /echo/i
-  abort 'The RDF command in the config file appears to be subject to command injection.  I will not continue'
+  abort 'The RDF command appears to be subject to command injection.  I will not continue'
 when !(/rdf$/ =~ $_)
   abort "this software requires that Kelloggs Distiller tool is used. The distiller command must end in 'rdf'"
 end
 RDF_COMMAND = rdf_command
 
-tika_command = CONFIG.dig(:tika, :command)
-tika_command ||= 'http://localhost:9998/meta'
+# tika_command = CONFIG.dig(:tika, :command)
+tika_command = ENV['TIKA_COMMAND'] || 'http://localhost:9998/meta'
 TIKA_COMMAND = tika_command

data/lib/fsp_harvester/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FspHarvester
-  VERSION = "0.1.15"
+  VERSION = "0.1.18"
 end

data/lib/harvester_utils.rb

@@ -4,8 +4,8 @@ module HarvesterTools
 
   class Utils
 
-    def self.resolve_guid(guid:)
-      @meta = HarvesterTools::MetadataObject.new
+    def self.resolve_guid(guid:, metadata: HarvesterTools::MetadataObject.new)
+      @meta = metadata
       @meta.all_uris = [guid]
       type, url = convertToURL(guid: guid)
       links = Array.new

data/lib/metadata_object.rb

@@ -1,8 +1,8 @@
 module HarvesterTools
   class MetadataObject
-    attr_accessor :hash, :graph, :comments, :links, :warnings, :guidtype, :full_response, :all_uris  # a hash of metadata # a RDF.rb graph of metadata  # an array of comments  # the type of GUID that was detected # will be an array of Net::HTTP::Response
+    attr_accessor :hash, :graph, :comments, :links, :warnings, :guidtype, :full_response, :all_uris, :guid, :score, :version, :date  # a hash of metadata # a RDF.rb graph of metadata  # an array of comments  # the type of GUID that was detected # will be an array of Net::HTTP::Response
 
-    def initialize(_params = {}) # get a name from the "new" call, or set a default
+    def initialize() # get a name from the "new" call, or set a default
       @hash = {}
       @graph = RDF::Graph.new
       @comments =  []
@@ -10,6 +10,10 @@ module HarvesterTools
       @full_response = []
       @links = []
       @all_uris = []
+      @guid = ""
+      @score = 0
+      @version = '0.0'
+      @date = Time.now.strftime('%Y-%m-%dT%H:%M:%S.%L%z')
       w = RestClient.get("https://raw.githubusercontent.com/markwilkinson/FAIR-Signposting-Harvester/master/lib/warnings.json")
       #@warn = File.read("./lib/warnings.json")
       @warn = JSON.parse(w)

data/lib/{config.conf_local → obselete_config.conf_docker}

@@ -2,7 +2,7 @@
 command="extruct"
 
 [rdf]
-command="/home/osboxes/.rvm/gems/ruby-3.0.0/bin/rdf"
+command="/usr/local/bundle/bin/rdf"
 
 [tika]
 command="http://tika:9998/meta"

data/lib/warnings.json

@@ -115,5 +115,12 @@
                   {"Documentation": "http://dx.doi.org/10.17487/RFC8259"},
                   {"Validator": "https://jsononline.net/json-validator"}],
       "severity": "WARN"
+   },
+   "600": {
+      "message": "Data identifier cannot be unambiguously determined, therefore cannot be tested against known persistent identifier schemas",
+      "linkout": [],
+      "severity": "FAILURE"
    }
+
+
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fsp_harvester
 version: !ruby/object:Gem::Version
-  version: 0.1.15
+  version: 0.1.18
 platform: ruby
 authors:
 - Mark Wilkinson
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-08-12 00:00:00.000000000 Z
+date: 2022-08-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -182,9 +182,6 @@ files:
 - bin/setup
 - example_test.rb
 - launch.json
-- lib/config.conf
-- lib/config.conf_docker
-- lib/config.conf_local
 - lib/constants.rb
 - lib/external_tools.rb
 - lib/fsp_harvester.rb
@@ -195,6 +192,9 @@ files:
 - lib/metadata_harvester.rb
 - lib/metadata_object.rb
 - lib/metadata_parser.rb
+- lib/obselete_config.conf
+- lib/obselete_config.conf_docker
+- lib/obselete_config.conf_local
 - lib/signposting_tests.rb
 - lib/warnings.json
 - lib/web_utils.rb