fsp_harvester 0.1.14 → 0.1.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e285f00da696d7e39d80df794be9524af6e63ea01deb4e73f6c30b3694c016ff
-  data.tar.gz: fb81b5c1c0fac3bb22e078663025855e5accdb355db1811a4687fb1bca54bc61
+  metadata.gz: 9e0ffd5048e360ce8e8cced890a586664af797065d2c8d6312927d694835e84b
+  data.tar.gz: 840269a8b28da70bed8c5e46674ff3730cbee66f624064cab84f98d5b3a2ff00
 SHA512:
-  metadata.gz: 194132eb78246291a3cb96566ca6a283841a0427afcd6a6abb79c590dbc2c54108e3e8cfef9e4802a77008f1a4c9c94ea7862987e81ce1b4b97cd1fdaf25ca23
-  data.tar.gz: 9765647726c2bfcd7e790ba11929d257610672bc92d8d11756824432e90db4c05036b2cfcede1a55da95f1e74b9e87fd078c78284c356897a5bdc0a17593a3a1
+  metadata.gz: 4c01cc88a8f57e024c7aeed89a8251d97b130bca987dc14d914e87fa87ea744d3de7ab11ca340b0456f295edafdd872d4f63d0f0ef23dbe9c3cc8ebc97a64ae5
+  data.tar.gz: 2c274758ec874bb1c25ebd5286ecbc2b7e91205430a94cf3ada9c7350511fe362532f2c6d213a6fd6657ccdf7184df9c0eaf9c2461c0d25dc87da00b2aded390

data/.rspec_status

@@ -42,14 +42,15 @@ example_id                         | status | run_time        |
 ./spec/describedby_spec.rb[1:1:15] | passed | 2.28 seconds    |
 ./spec/fsp_harvester_spec.rb[1:1]  | passed | 0.00058 seconds |
 ./spec/fsp_harvester_spec.rb[1:2]  | failed | 2.92 seconds    |
-./spec/item_spec.rb[1:1:1]         | passed | 2.94 seconds    |
-./spec/item_spec.rb[1:1:2]         | passed | 3 seconds       |
-./spec/item_spec.rb[1:1:3]         | passed | 1.35 seconds    |
-./spec/item_spec.rb[1:1:4]         | passed | 1.83 seconds    |
-./spec/item_spec.rb[1:1:5]         | passed | 2.26 seconds    |
-./spec/item_spec.rb[1:1:6]         | passed | 2.17 seconds    |
-./spec/item_spec.rb[1:1:7]         | passed | 2.8 seconds     |
-./spec/item_spec.rb[1:1:8]         | passed | 0.52869 seconds |
+./spec/fsp_harvester_spec.rb[1:3]  | passed | 6.87 seconds    |
+./spec/item_spec.rb[1:1:1]         | passed | 3.19 seconds    |
+./spec/item_spec.rb[1:1:2]         | passed | 2.81 seconds    |
+./spec/item_spec.rb[1:1:3]         | passed | 1.27 seconds    |
+./spec/item_spec.rb[1:1:4]         | passed | 1.76 seconds    |
+./spec/item_spec.rb[1:1:5]         | passed | 2.43 seconds    |
+./spec/item_spec.rb[1:1:6]         | passed | 2.23 seconds    |
+./spec/item_spec.rb[1:1:7]         | passed | 2.94 seconds    |
+./spec/item_spec.rb[1:1:8]         | passed | 0.52517 seconds |
 ./spec/type_spec.rb[1:1:1]         | passed | 1.35 seconds    |
 ./spec/type_spec.rb[1:1:2]         | passed | 1.32 seconds    |
 ./spec/type_spec.rb[1:1:3]         | passed | 1.54 seconds    |

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fsp_harvester (0.1.14)
+    fsp_harvester (0.1.15)
       json (~> 2.0)
       linkeddata (~> 3.2)
       linkheaders-processor (~> 0.1.17)

data/lib/fsp_harvester/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FspHarvester
-  VERSION = "0.1.14"
+  VERSION = "0.1.15"
 end

data/lib/harvester_brute.rb

@@ -0,0 +1,48 @@
+module HarvesterTools
+  class Error < StandardError
+  end
+
+  class BruteForce
+    def self.begin_brute_force(guid:, metadata: HarvesterTools::MetadataObject.new)
+      type, url = HarvesterTools::Utils.convertToURL(guid: guid)
+      return false unless type
+
+      do_content_negotiation(url: url, metadata: metadata)
+      metadata
+    end
+
+    def self.do_content_negotiation(url:, metadata:)
+      response = resolve_url_brute(url: url, metadata: metadata, headers: ACCEPT_ALL_HEADER)
+      if response
+        HarvesterTools::MetadataHarvester.extract_metadata_from_body(response: response, metadata: metadata)
+      end
+      response = resolve_url_brute(url: url, metadata: metadata, headers: ACCEPT_STAR_HEADER)
+      if response
+        HarvesterTools::MetadataHarvester.extract_metadata_from_body(response: response, metadata: metadata) # extract from landing page
+        response = resolve_url_brute(url: response.request.url, metadata: metadata, headers: ACCEPT_ALL_HEADER) # now do content negotiation on the landing page
+        if response
+          HarvesterTools::MetadataHarvester.extract_metadata_from_body(response: response, metadata: metadata) # extract from landing page
+        end
+      end
+    end
+
+    def self.resolve_url_brute(url:, method: :get, nolinkheaders: true, headers:, metadata:)
+      @meta = metadata
+      @meta.guidtype = 'uri' if @meta.guidtype.nil?
+      warn "\n\n BRUTE FETCHING #{url} #{headers}\n\n"
+      response = HarvesterTools::WebUtils.fspfetch(url: url, headers: headers, method: method, meta: @meta)
+      warn "\n\n head #{response.headers.inspect}\n\n" if response
+
+      unless response
+        @meta.add_warning(['001', url, headers])
+        @meta.comments << "WARN: Unable to resolve #{url} using HTTP Accept header #{headers}.\n"
+        @meta.full_response << [url, "No response"]
+        false
+      end
+
+      @meta.comments << "INFO: following redirection using this header led to the following URL: #{@meta.all_uris.last}.  Using the output from this URL for the next few tests..."
+      @meta.full_response << [url, response.body]
+      response
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fsp_harvester
 version: !ruby/object:Gem::Version
-  version: 0.1.14
+  version: 0.1.15
 platform: ruby
 authors:
 - Mark Wilkinson
@@ -190,6 +190,7 @@ files:
 - lib/fsp_harvester.rb
 - lib/fsp_harvester/version.rb
 - lib/harvester.rb
+- lib/harvester_brute.rb
 - lib/harvester_utils.rb
 - lib/metadata_harvester.rb
 - lib/metadata_object.rb