fsp_harvester 0.1.14 → 0.1.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e285f00da696d7e39d80df794be9524af6e63ea01deb4e73f6c30b3694c016ff
-  data.tar.gz: fb81b5c1c0fac3bb22e078663025855e5accdb355db1811a4687fb1bca54bc61
+  metadata.gz: c3f2b3409b575db21edc69a7e0e4bfbf5be09734fbcc4f4b0d5accb5fedad6c2
+  data.tar.gz: 27813b4e090515a869d5fbc519a717eb06e7bc0559e42d33d1284093758f229f
 SHA512:
-  metadata.gz: 194132eb78246291a3cb96566ca6a283841a0427afcd6a6abb79c590dbc2c54108e3e8cfef9e4802a77008f1a4c9c94ea7862987e81ce1b4b97cd1fdaf25ca23
-  data.tar.gz: 9765647726c2bfcd7e790ba11929d257610672bc92d8d11756824432e90db4c05036b2cfcede1a55da95f1e74b9e87fd078c78284c356897a5bdc0a17593a3a1
+  metadata.gz: 33aeec82ef754f219db35eba08e68e35f0aecc570b2769f943f073cf1c4b5a9cdfed912c7fc40261797e4d3b00d7f668228aa5cd442eec213c272455aab1a275
+  data.tar.gz: 80f03c769794a7bf8054d95c667274a3a3ddde26242b0969e15511f78b74cdd5d20508ed7c36e47a446012ff021f0f3ad5a9aa4a44dc91d07c8c86d0da35c59e

data/.rspec_status

@@ -40,16 +40,17 @@ example_id                         | status | run_time        |
 ./spec/describedby_spec.rb[1:1:13] | passed | 1.74 seconds    |
 ./spec/describedby_spec.rb[1:1:14] | passed | 2.27 seconds    |
 ./spec/describedby_spec.rb[1:1:15] | passed | 2.28 seconds    |
-./spec/fsp_harvester_spec.rb[1:1]  | passed | 0.00058 seconds |
-./spec/fsp_harvester_spec.rb[1:2]  | failed | 2.92 seconds    |
-./spec/item_spec.rb[1:1:1]         | passed | 2.94 seconds    |
-./spec/item_spec.rb[1:1:2]         | passed | 3 seconds       |
-./spec/item_spec.rb[1:1:3]         | passed | 1.35 seconds    |
-./spec/item_spec.rb[1:1:4]         | passed | 1.83 seconds    |
-./spec/item_spec.rb[1:1:5]         | passed | 2.26 seconds    |
-./spec/item_spec.rb[1:1:6]         | passed | 2.17 seconds    |
-./spec/item_spec.rb[1:1:7]         | passed | 2.8 seconds     |
-./spec/item_spec.rb[1:1:8]         | passed | 0.52869 seconds |
+./spec/fsp_harvester_spec.rb[1:1]  | passed | 0.00693 seconds |
+./spec/fsp_harvester_spec.rb[1:2]  | passed | 3.65 seconds    |
+./spec/fsp_harvester_spec.rb[1:3]  | passed | 9.96 seconds    |
+./spec/item_spec.rb[1:1:1]         | passed | 3.19 seconds    |
+./spec/item_spec.rb[1:1:2]         | passed | 2.81 seconds    |
+./spec/item_spec.rb[1:1:3]         | passed | 1.27 seconds    |
+./spec/item_spec.rb[1:1:4]         | passed | 1.76 seconds    |
+./spec/item_spec.rb[1:1:5]         | passed | 2.43 seconds    |
+./spec/item_spec.rb[1:1:6]         | passed | 2.23 seconds    |
+./spec/item_spec.rb[1:1:7]         | passed | 2.94 seconds    |
+./spec/item_spec.rb[1:1:8]         | passed | 0.52517 seconds |
 ./spec/type_spec.rb[1:1:1]         | passed | 1.35 seconds    |
 ./spec/type_spec.rb[1:1:2]         | passed | 1.32 seconds    |
 ./spec/type_spec.rb[1:1:3]         | passed | 1.54 seconds    |

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fsp_harvester (0.1.14)
+    fsp_harvester (0.1.17)
       json (~> 2.0)
       linkeddata (~> 3.2)
       linkheaders-processor (~> 0.1.17)

data/README.md

@@ -20,6 +20,39 @@ Or install it yourself as:
 
 ## Usage
 
+```
+require 'fsp_harvester'
+
+ENV['EXTRUCT_COMMAND'] = "extruct"
+ENV['RDF_COMMAND'] = "/home/user/.rvm/gems/ruby-3.0.0/bin/rdf" # kelloggs distiller
+ENV['TIKA_COMMAND'] = "http://localhost:9998/meta" # assumes using the docker version of tika
+
+# to only follow the FAIR signposting specification:
+links, metadata = HarvesterTools::Utils.resolve_guid(guid: guid)
+
+links.each do |link|
+    puts link.href
+    puts link.relation
+end
+
+# note, you don't need to catch the return value here.  The metadata object that is passed in will be modified
+metadata = FspHarvester::Utils.gather_metadata_from_describedby_links(links: links, metadata: metadata)
+
+linkeddata = metadata.graph
+hashdata = metadata.hash
+comments = metadata.comments
+warnings = metadata.warnings
+
+# if you want to try other things like content negotiation and "scraping" from HTML, do this:
+# note, you don't need to catch the return value here.  The metadata object that is passed in will be modified
+metadata = HarvesterTools::BruteForce.begin_brute_force(guid: guid, metadata: metadata)
+
+linkeddata = metadata.graph
+hashdata = metadata.hash
+comments = metadata.comments
+warnings = metadata.warnings
+
+```
 
 
 ## Development

data/lib/constants.rb

@@ -78,31 +78,31 @@ GUID_TYPES = {
   'ark' => Regexp.new(%r{^ark:/[^\s]+$}) 
 }
 
-CONFIG = File.exist?('config.conf') ? ParseConfig.new('config.conf') : {}
-extruct = CONFIG.dig(:extruct, :command)
-extruct ||= 'extruct'
+# CONFIG = File.exist?('config.conf') ? ParseConfig.new('config.conf') : {}
+# extruct = CONFIG.dig(:extruct, :command)
+extruct = ENV['EXTRUCT_COMMAND'] || 'extruct'
 extruct.strip!
 case extruct
 when /[&|;`$\s]/
-  abort 'The Extruct command in the config file appears to be subject to command injection.  I will not continue'
+  abort 'The Extruct command appears to be subject to command injection.  I will not continue'
 when /echo/i
-  abort 'The Extruct command in the config file appears to be subject to command injection.  I will not continue'
+  abort 'The Extruct command appears to be subject to command injection.  I will not continue'
 end
 EXTRUCT_COMMAND = extruct
 
-rdf_command = CONFIG.dig(:rdf, :command)
-rdf_command ||= 'rdf'
+# rdf_command = CONFIG.dig(:rdf, :command)
+rdf_command = ENV['RDF_COMMAND'] || 'rdf'
 rdf_command.strip
 case rdf_command
 when /[&|;`$\s]/
-  abort 'The RDF command in the config file appears to be subject to command injection.  I will not continue'
+  abort 'The RDF command appears to be subject to command injection.  I will not continue'
 when /echo/i
-  abort 'The RDF command in the config file appears to be subject to command injection.  I will not continue'
+  abort 'The RDF command appears to be subject to command injection.  I will not continue'
 when !(/rdf$/ =~ $_)
   abort "this software requires that Kelloggs Distiller tool is used. The distiller command must end in 'rdf'"
 end
 RDF_COMMAND = rdf_command
 
-tika_command = CONFIG.dig(:tika, :command)
-tika_command ||= 'http://localhost:9998/meta'
+# tika_command = CONFIG.dig(:tika, :command)
+tika_command = ENV['TIKA_COMMAND'] || 'http://localhost:9998/meta'
 TIKA_COMMAND = tika_command

data/lib/fsp_harvester/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FspHarvester
-  VERSION = "0.1.14"
+  VERSION = "0.1.17"
 end

data/lib/harvester_brute.rb

@@ -0,0 +1,48 @@
+module HarvesterTools
+  class Error < StandardError
+  end
+
+  class BruteForce
+    def self.begin_brute_force(guid:, metadata: HarvesterTools::MetadataObject.new)
+      type, url = HarvesterTools::Utils.convertToURL(guid: guid)
+      return false unless type
+
+      do_content_negotiation(url: url, metadata: metadata)
+      metadata
+    end
+
+    def self.do_content_negotiation(url:, metadata:)
+      response = resolve_url_brute(url: url, metadata: metadata, headers: ACCEPT_ALL_HEADER)
+      if response
+        HarvesterTools::MetadataHarvester.extract_metadata_from_body(response: response, metadata: metadata)
+      end
+      response = resolve_url_brute(url: url, metadata: metadata, headers: ACCEPT_STAR_HEADER)
+      if response
+        HarvesterTools::MetadataHarvester.extract_metadata_from_body(response: response, metadata: metadata) # extract from landing page
+        response = resolve_url_brute(url: response.request.url, metadata: metadata, headers: ACCEPT_ALL_HEADER) # now do content negotiation on the landing page
+        if response
+          HarvesterTools::MetadataHarvester.extract_metadata_from_body(response: response, metadata: metadata) # extract from landing page
+        end
+      end
+    end
+
+    def self.resolve_url_brute(url:, method: :get, nolinkheaders: true, headers:, metadata:)
+      @meta = metadata
+      @meta.guidtype = 'uri' if @meta.guidtype.nil?
+      warn "\n\n BRUTE FETCHING #{url} #{headers}\n\n"
+      response = HarvesterTools::WebUtils.fspfetch(url: url, headers: headers, method: method, meta: @meta)
+      warn "\n\n head #{response.headers.inspect}\n\n" if response
+
+      unless response
+        @meta.add_warning(['001', url, headers])
+        @meta.comments << "WARN: Unable to resolve #{url} using HTTP Accept header #{headers}.\n"
+        @meta.full_response << [url, "No response"]
+        false
+      end
+
+      @meta.comments << "INFO: following redirection using this header led to the following URL: #{@meta.all_uris.last}.  Using the output from this URL for the next few tests..."
+      @meta.full_response << [url, response.body]
+      response
+    end
+  end
+end

data/lib/harvester_utils.rb

@@ -4,8 +4,8 @@ module HarvesterTools
 
   class Utils
 
-    def self.resolve_guid(guid:)
-      @meta = HarvesterTools::MetadataObject.new
+    def self.resolve_guid(guid:, metadata: HarvesterTools::MetadataObject.new)
+      @meta = metadata
       @meta.all_uris = [guid]
       type, url = convertToURL(guid: guid)
       links = Array.new

data/lib/metadata_object.rb

@@ -1,6 +1,6 @@
 module HarvesterTools
   class MetadataObject
-    attr_accessor :hash, :graph, :comments, :links, :warnings, :guidtype, :full_response, :all_uris  # a hash of metadata # a RDF.rb graph of metadata  # an array of comments  # the type of GUID that was detected # will be an array of Net::HTTP::Response
+    attr_accessor :hash, :graph, :comments, :links, :warnings, :guidtype, :full_response, :all_uris, :guid, :score  # a hash of metadata # a RDF.rb graph of metadata  # an array of comments  # the type of GUID that was detected # will be an array of Net::HTTP::Response
 
     def initialize(_params = {}) # get a name from the "new" call, or set a default
       @hash = {}
@@ -10,6 +10,8 @@ module HarvesterTools
       @full_response = []
       @links = []
       @all_uris = []
+      @guid = ""
+      @score = 0
       w = RestClient.get("https://raw.githubusercontent.com/markwilkinson/FAIR-Signposting-Harvester/master/lib/warnings.json")
       #@warn = File.read("./lib/warnings.json")
       @warn = JSON.parse(w)

data/lib/{config.conf_local → obselete_config.conf_docker}

@@ -2,7 +2,7 @@
 command="extruct"
 
 [rdf]
-command="/home/osboxes/.rvm/gems/ruby-3.0.0/bin/rdf"
+command="/usr/local/bundle/bin/rdf"
 
 [tika]
 command="http://tika:9998/meta"

data/lib/warnings.json

@@ -115,5 +115,12 @@
                   {"Documentation": "http://dx.doi.org/10.17487/RFC8259"},
                   {"Validator": "https://jsononline.net/json-validator"}],
       "severity": "WARN"
+   },
+   "600": {
+      "message": "Data identifier cannot be unambiguously determined, therefore cannot be tested against known persistent identifier schemas",
+      "linkout": [],
+      "severity": "FAILURE"
    }
+
+
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fsp_harvester
 version: !ruby/object:Gem::Version
-  version: 0.1.14
+  version: 0.1.17
 platform: ruby
 authors:
 - Mark Wilkinson
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-08-12 00:00:00.000000000 Z
+date: 2022-08-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -182,18 +182,19 @@ files:
 - bin/setup
 - example_test.rb
 - launch.json
-- lib/config.conf
-- lib/config.conf_docker
-- lib/config.conf_local
 - lib/constants.rb
 - lib/external_tools.rb
 - lib/fsp_harvester.rb
 - lib/fsp_harvester/version.rb
 - lib/harvester.rb
+- lib/harvester_brute.rb
 - lib/harvester_utils.rb
 - lib/metadata_harvester.rb
 - lib/metadata_object.rb
 - lib/metadata_parser.rb
+- lib/obselete_config.conf
+- lib/obselete_config.conf_docker
+- lib/obselete_config.conf_local
 - lib/signposting_tests.rb
 - lib/warnings.json
 - lib/web_utils.rb