fsp_harvester 0.1.13 → 0.1.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '099b2769aca02c9b6fba26583dfbcfc4b60c39798be4adf0a4c71a989af0094c'
-  data.tar.gz: 80b8657befce11cdd8d58c420fe8039ba5407b29b6e0fed645325212de95f4d0
+  metadata.gz: aec11fd57963ffb176ddb88338b9e262027c9a7d39364089ae130fb4b628bf5b
+  data.tar.gz: f8733a00de5c6c24a622235c18ba0dae208f5bac52d50607480e51fd563678c8
 SHA512:
-  metadata.gz: 683362c6a0710bf9a0d5420a9ae6fe8372338f9c6132ae5ae56619dce1bfa88df7914008a661cf49685000166a0c4c6b476691acda109326c3955f73b796cc4e
-  data.tar.gz: 931972a9f872bcb90e11ed731e4de9b406b9c3ccb1f44e6af250d9cb817616040e86149d34f070fe9ac0c6711f37438922b0efb5e2915ce00e39fa4ce09c030b
+  metadata.gz: c484e41aa0305f34d0bf7f82cad60b9b02106ffe80b9371c99e77b199eef9ce52818222368b8b3a3ff73d94dba89b8d7fb815d29c95ca335772946e1e9762849
+  data.tar.gz: '09dfdcc12b9176bc88c31a196893ae9ede6c35c2fd59271ca8fa5b1c29f0807ee82c8416ca5f8b7ff75c6caab71648b6d0e15d0976784cb7d34ff8686332be37'

data/.rspec_status

@@ -42,14 +42,15 @@ example_id                         | status | run_time        |
 ./spec/describedby_spec.rb[1:1:15] | passed | 2.28 seconds    |
 ./spec/fsp_harvester_spec.rb[1:1]  | passed | 0.00058 seconds |
 ./spec/fsp_harvester_spec.rb[1:2]  | failed | 2.92 seconds    |
-./spec/item_spec.rb[1:1:1]         | passed | 2.94 seconds    |
-./spec/item_spec.rb[1:1:2]         | passed | 3 seconds       |
-./spec/item_spec.rb[1:1:3]         | passed | 1.35 seconds    |
-./spec/item_spec.rb[1:1:4]         | passed | 1.83 seconds    |
-./spec/item_spec.rb[1:1:5]         | passed | 2.26 seconds    |
-./spec/item_spec.rb[1:1:6]         | passed | 2.17 seconds    |
-./spec/item_spec.rb[1:1:7]         | passed | 2.8 seconds     |
-./spec/item_spec.rb[1:1:8]         | passed | 0.52869 seconds |
+./spec/fsp_harvester_spec.rb[1:3]  | passed | 6.87 seconds    |
+./spec/item_spec.rb[1:1:1]         | passed | 3.19 seconds    |
+./spec/item_spec.rb[1:1:2]         | passed | 2.81 seconds    |
+./spec/item_spec.rb[1:1:3]         | passed | 1.27 seconds    |
+./spec/item_spec.rb[1:1:4]         | passed | 1.76 seconds    |
+./spec/item_spec.rb[1:1:5]         | passed | 2.43 seconds    |
+./spec/item_spec.rb[1:1:6]         | passed | 2.23 seconds    |
+./spec/item_spec.rb[1:1:7]         | passed | 2.94 seconds    |
+./spec/item_spec.rb[1:1:8]         | passed | 0.52517 seconds |
 ./spec/type_spec.rb[1:1:1]         | passed | 1.35 seconds    |
 ./spec/type_spec.rb[1:1:2]         | passed | 1.32 seconds    |
 ./spec/type_spec.rb[1:1:3]         | passed | 1.54 seconds    |

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fsp_harvester (0.1.13)
+    fsp_harvester (0.1.16)
       json (~> 2.0)
       linkeddata (~> 3.2)
       linkheaders-processor (~> 0.1.17)

data/lib/config.conf

@@ -2,7 +2,7 @@
 command="extruct"
 
 [rdf]
-command="/home/osboxes/.rvm/gems/ruby-3.0.0/bin/rdf"
+command="/usr/local/bundle/bin/rdf"
 
 [tika]
 command="http://tika:9998/meta"

data/lib/fsp_harvester/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FspHarvester
-  VERSION = "0.1.13"
+  VERSION = "0.1.16"
 end

data/lib/harvester_brute.rb

@@ -0,0 +1,48 @@
+module HarvesterTools
+  class Error < StandardError
+  end
+
+  class BruteForce
+    def self.begin_brute_force(guid:, metadata: HarvesterTools::MetadataObject.new)
+      type, url = HarvesterTools::Utils.convertToURL(guid: guid)
+      return false unless type
+
+      do_content_negotiation(url: url, metadata: metadata)
+      metadata
+    end
+
+    def self.do_content_negotiation(url:, metadata:)
+      response = resolve_url_brute(url: url, metadata: metadata, headers: ACCEPT_ALL_HEADER)
+      if response
+        HarvesterTools::MetadataHarvester.extract_metadata_from_body(response: response, metadata: metadata)
+      end
+      response = resolve_url_brute(url: url, metadata: metadata, headers: ACCEPT_STAR_HEADER)
+      if response
+        HarvesterTools::MetadataHarvester.extract_metadata_from_body(response: response, metadata: metadata) # extract from landing page
+        response = resolve_url_brute(url: response.request.url, metadata: metadata, headers: ACCEPT_ALL_HEADER) # now do content negotiation on the landing page
+        if response
+          HarvesterTools::MetadataHarvester.extract_metadata_from_body(response: response, metadata: metadata) # extract from landing page
+        end
+      end
+    end
+
+    def self.resolve_url_brute(url:, method: :get, nolinkheaders: true, headers:, metadata:)
+      @meta = metadata
+      @meta.guidtype = 'uri' if @meta.guidtype.nil?
+      warn "\n\n BRUTE FETCHING #{url} #{headers}\n\n"
+      response = HarvesterTools::WebUtils.fspfetch(url: url, headers: headers, method: method, meta: @meta)
+      warn "\n\n head #{response.headers.inspect}\n\n" if response
+
+      unless response
+        @meta.add_warning(['001', url, headers])
+        @meta.comments << "WARN: Unable to resolve #{url} using HTTP Accept header #{headers}.\n"
+        @meta.full_response << [url, "No response"]
+        false
+      end
+
+      @meta.comments << "INFO: following redirection using this header led to the following URL: #{@meta.all_uris.last}.  Using the output from this URL for the next few tests..."
+      @meta.full_response << [url, response.body]
+      response
+    end
+  end
+end

data/lib/web_utils.rb

@@ -18,13 +18,13 @@ module HarvesterTools
         warn "final URL #{response.request.url}"
         warn "Response code #{response.code}"
         if response.code == 203 
-          meta.warnings << ["002", url, headers]
+          meta.add_warning(["002", url, headers])
           meta.comments << "WARN: Response is non-authoritative (HTTP response code: #{response.code}).  Headers may have been manipulated encountered when trying to resolve #{url}\n"
         end
         response
       rescue RestClient::ExceptionWithResponse => e
         warn "EXCEPTION WITH RESPONSE! #{e.response.code} with response #{e.response}\nfailed response headers: #{e.response.headers}"
-        meta.warnings << ["003", url, headers] 
+        meta.add_warning(["003", url, headers])
         meta.comments << "WARN: HTTP error #{e} encountered when trying to resolve #{url}\n"
         if (e.response.code == 500 or e.response.code == 404)
           return false
@@ -34,13 +34,13 @@ module HarvesterTools
         # now we are returning the headers and body that were returned
       rescue RestClient::Exception => e
         warn "EXCEPTION WITH NO RESPONSE! #{e}"
-        meta.warnings << ["003", url, headers]
+        meta.add_warning(["003", url, headers])
         meta.comments << "WARN: HTTP error #{e} encountered when trying to resolve #{url}\n"
         false
         # now we are returning 'False', and we will check that with an \"if\" statement in our main code
       rescue Exception => e
         warn "EXCEPTION UNKNOWN! #{e}"
-        meta.warnings << ["003", url, headers]
+        meta.add_warning(["003", url, headers])
         meta.comments << "WARN: HTTP error #{e} encountered when trying to resolve #{url}\n"
         false
         # now we are returning 'False', and we will check that with an \"if\" statement in our main code

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fsp_harvester
 version: !ruby/object:Gem::Version
-  version: 0.1.13
+  version: 0.1.16
 platform: ruby
 authors:
 - Mark Wilkinson
@@ -190,6 +190,7 @@ files:
 - lib/fsp_harvester.rb
 - lib/fsp_harvester/version.rb
 - lib/harvester.rb
+- lib/harvester_brute.rb
 - lib/harvester_utils.rb
 - lib/metadata_harvester.rb
 - lib/metadata_object.rb