linkheaders-processor 0.1.17 → 0.1.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b10f24c0498058f393a2142465f0ff2e559dc2a8721ea8cf6a90178c8ff21789
-  data.tar.gz: 5cd760d37f6e82f63cb8271375bec72364264173010fda86e1ae32f1a424642d
+  metadata.gz: 65747de845763341178717385337c65ed5d78a7df57bccecee17f392d8330b12
+  data.tar.gz: 1e2c5ae203200e40e8fc238b211f63491bc2ba4756d3bd807ccb916a7c9b270f
 SHA512:
-  metadata.gz: e696571f2c9da932ff461af46740824de0728a6accd327ab6e109ab84530d313868d120cff5a62f670655efe6531069d7b6a358f1dcfcb94560cdd647614ff35
-  data.tar.gz: cf056d618d352bbcfaa43e59317fbc6f739186ac3e7ae26534e8f12721dec0d7ab2e8edfa07a2434e881a37d1cd18bf30e84f6cfac402fcc74ecd8c86170fa73
+  metadata.gz: a92597e6f649e5abdc524862c051a0a2d442c753976ea5cea70ce0ce4b1c30261d0f0f3e9e3e577ad55c396cad7a780c806f068012ef26a3ae91a7cc8fbb109e
+  data.tar.gz: bdd96b086e950c0b427ec8d15ed151723639d72b81d9f3a15adfca22bf7326c51ee9711cf0aa451a3e1d472cb14d8c46d6ced944627058d5e87fa695dbcde6f6

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    linkheaders-processor (0.1.17)
+    linkheaders-processor (0.1.19)
       json (~> 2.0)
       json-ld (~> 3.2)
       json-ld-preloaded (~> 3.2)
@@ -13,8 +13,8 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.8.0)
-      public_suffix (>= 2.0.2, < 5.0)
+    addressable (2.8.5)
+      public_suffix (>= 2.0.2, < 6.0)
     ast (2.4.2)
     diff-lcs (1.5.0)
     domain_name (0.5.20190701)
@@ -39,7 +39,7 @@ GEM
     faraday-encoding (0.0.5)
       faraday
     faraday-excon (1.1.0)
-    faraday-http-cache (2.4.0)
+    faraday-http-cache (2.5.0)
       faraday (>= 0.8)
     faraday-httpclient (1.0.1)
     faraday-multipart (1.0.4)
@@ -51,21 +51,21 @@ GEM
     faraday-retry (1.0.3)
     faraday_middleware (1.2.0)
       faraday (~> 1.0)
-    fastimage (2.2.6)
+    fastimage (2.2.7)
     htmlentities (4.3.4)
     http-accept (1.7.0)
     http-cookie (1.0.5)
       domain_name (~> 0.5)
     json (2.6.2)
-    json-canonicalization (0.3.0)
-    json-ld (3.2.3)
+    json-canonicalization (0.3.2)
+    json-ld (3.2.5)
       htmlentities (~> 4.3)
-      json-canonicalization (~> 0.3)
+      json-canonicalization (~> 0.3, >= 0.3.2)
       link_header (~> 0.0, >= 0.0.8)
       multi_json (~> 1.15)
-      rack (~> 2.2)
-      rdf (~> 3.2, >= 3.2.9)
-    json-ld-preloaded (3.2.0)
+      rack (>= 2.2, < 4)
+      rdf (~> 3.2, >= 3.2.10)
+    json-ld-preloaded (3.2.2)
       json-ld (~> 3.2)
       rdf (~> 3.2)
     link_header (0.0.8)
@@ -79,24 +79,24 @@ GEM
       fastimage (~> 2.2)
       nesty (~> 1.0)
       nokogiri (~> 1.11)
-    mime-types (3.4.1)
+    mime-types (3.5.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2022.0105)
+    mime-types-data (3.2023.0808)
     multi_json (1.15.0)
     multipart-post (2.2.3)
     nesty (1.0.2)
     netrc (0.11.0)
-    nokogiri (1.13.8-x86_64-linux)
+    nokogiri (1.15.4-x86_64-linux)
       racc (~> 1.4)
     parallel (1.22.1)
     parser (3.1.2.0)
       ast (~> 2.4.1)
-    public_suffix (4.0.7)
-    racc (1.6.0)
-    rack (2.2.4)
+    public_suffix (5.0.3)
+    racc (1.7.1)
+    rack (3.0.8)
     rainbow (3.1.1)
     rake (13.0.6)
-    rdf (3.2.9)
+    rdf (3.2.11)
       link_header (~> 0.0, >= 0.0.8)
     regexp_parser (2.5.0)
     rest-client (2.1.0)

data/lib/linkheaders/link.rb

@@ -127,7 +127,7 @@ module LinkHeaders
           if l.relation != link.relation
             @warnings |= ['WARN: Found identical hrefs with different relation types.  This may be suspicious. Both have been retained']
           else
-            @warnings |= ['WARN: found apparent duplicate. Ignoring and returning known link']
+            @warnings |= ["WARN: found apparent duplicate #{l.relation} #{l.href} EQUALS#{link.href}. Ignoring and returning known link #{l.relation} #{l.href}"]
             link = l
           end
         end

data/lib/linkheaders/processor/version.rb

@@ -3,6 +3,6 @@
 
 module LinkHeaders
   class Processor
-    VERSION = "0.1.17"
+    VERSION = "0.1.19"
   end
 end

data/lib/linkheaders/processor.rb

@@ -59,14 +59,14 @@ module LinkHeaders
         return [[], []]
       end
 
-      newlinks = parse_http_link_headers(head) # pass guid to check against anchors in linksets
-      warn "HTTPlinks #{newlinks.inspect}"
+      _newlinks = parse_http_link_headers(head)
+      # warn "HTTPlinks #{newlinks.inspect}"
 
-      ['text/html','text/xhtml+xml', 'application/xhtml+xml'].each do |format|
+      ['text/html', 'text/xhtml+xml', 'application/xhtml+xml'].each do |format|
         if head[:content_type] and head[:content_type].match(format)
           warn "found #{format} content - parsing"
-          htmllinks = parse_html_link_headers(body: body, anchor: default_anchor) # pass html body to find HTML link headers
-          warn "htmllinks #{htmllinks.inspect}"
+          _htmllinks = parse_html_link_headers(body: body, anchor: default_anchor) # pass html body to find HTML link headers
+          # warn "htmllinks #{htmllinks.inspect}"
         end
       end
     end
@@ -99,10 +99,12 @@ module LinkHeaders
         # warn "link is:  #{part}"
 
         section = part.split(';') # ["<https://example.one.com>", "rel='preconnect'"]
-        # warn section
+        warn section
         next unless section[0]
 
         href = section[0][/<(.*)>/, 1]
+        next unless href  # this is mandatory!
+
         next unless section[1]
 
         sections = {}
@@ -123,10 +125,11 @@ module LinkHeaders
         relation = sections['rel']
         sections.delete('rel')
         relations = relation.split(/\s+/)  # handle the multiple relation case
-        warn "RELATIONS #{relations}"
+        # warn "HEADERS RELATIONS #{relations}"
 
         relations.each do |rel|
           next unless rel.match?(/\w/)
+          puts "LICENCE is #{href}\n\n" if rel == "license"
           newlinks << factory.new_link(responsepart: :header, anchor: anchor, href: href, relation: rel, **sections) # parsed['https://example.one.com'][:rel] = "preconnect"
         end
       end
@@ -143,7 +146,6 @@ module LinkHeaders
       # an array of elements that look like this: [{:rel=>"alternate", :type=>"application/ld+json", :href=>"http://scidata.vitk.lv/dataset/303.jsonld"}]
       newlinks = Array.new
       m.head_links.each do |l|
-        warn "HTML head link is:  #{l.inspect}"
         next unless l[:href] and l[:rel] # required
 
         anchor = l[:anchor] || default_anchor
@@ -154,7 +156,7 @@ module LinkHeaders
         l.delete(:href)        
         
         relations = relation.split(/\s+/)  # handle the multiple relation case
-        warn "RELATIONS #{relations}"
+        # warn "BODY RELATIONS #{relations}"
 
         relations.each do |rel|
           next unless rel.match?(/\w/)
@@ -189,9 +191,10 @@ module LinkHeaders
 
     def processJSONLinkset(href:)
       _headers, linkset = lhfetch(href, { 'Accept' => 'application/linkset+json' })
-      # warn "Linkset body #{linkset.inspect}"
+      # warn "Linkset body #{linkset.inspect}\n\nLinkset headers #{_headers}\n\n"
       newlinks = Array.new
       return nil unless linkset
+      # warn "linkset #{linkset}"
 
       # linkset = '{ "linkset":
       #   [
@@ -208,20 +211,28 @@ module LinkHeaders
       # }'
 
       linkset = JSON.parse(linkset)
+      # warn "linkset #{linkset}"
+      if linkset['data'] and linkset['data']['linkset']
+        linkset['linkset'] = linkset['data']['linkset']
+      end
+      return nil unless linkset['linkset'].first
       linkset['linkset'].each do |ls|
         # warn ls.inspect, "\n"
         anchor = ls['anchor'] || @default_anchor
-        ls.delete('anchor') if ls['anchor'] # we need to delete since all others have a list as a value
+        ls.delete('anchor') if ls['anchor'] # we need to delete since almost all others have a list as a value
         attrhash = {}
         # warn ls.keys, "\n"
 
-        ls.each_key do |relation| # key =  e.g. "item", "described-by". "cite"
-          # warn reltype, "\n"
+        ls.each_key do |relation| # relation =  e.g. "item", "described-by". "cite"
+          href = ""
+          # warn relation
           # warn ls[reltype], "\n"
+          ls[relation] = [ls[relation]] unless ls[relation].is_a? Array  # force it to be a list, if it isn't
           ls[relation].each do |attrs|  # attr = e.g.  {"href": "http://example.com/foo1", "type": "text/html"}
+            # warn "ATTR: #{attrs}"
             next unless attrs['href']  # this is a required attribute of a  linkset relation
-
             href = attrs['href']
+            attrs.delete("href")
             # now go through the other attributes of that relation
             attrs.each do |attr, val| # attr = e.g. "type"; val = "text/html"
               attrhash[attr.to_sym] = val
@@ -229,7 +240,6 @@ module LinkHeaders
           end
 
           relations = relation.split(/\s+/)  # handle the multiple relation case
-
           relations.each do |rel|
             next unless rel.match?(/\w/)
             newlinks << factory.new_link(responsepart: :header, anchor: anchor, href: href, relation: rel, **attrhash) # parsed['https://example.one.com'][:rel] = "preconnect"
@@ -245,13 +255,13 @@ module LinkHeaders
       # warn "linkset body #{linkset.inspect}"
       return {} unless linkset
 
-      links = linkset.scan(/(<.*?>[^<]+)/) # split on the open angle bracket, which indicates a new link
+#      links = linkset.scan(/(<.*?>[^<]+)/) # split on the open angle bracket, which indicates a new link
+      links = linkset.split(/,\n*/) # split on the comma+newline
       # warn "Links found #{links}"
 
       links.each do |ls|
-        # warn "workking on link #{ls}"
-        ls = ls.first # ls is a single element array
-        elements = ls.split(';') # semicolon delimited fields
+        # warn "working on link #{ls}"
+        elements = ls.split(';').map {|element| element.strip!} # semicolon delimited fields
         # ["<https://w3id.org/a2a-fair-metrics/08-http-describedby-citeas-linkset-txt/>", "anchor=\"https://s11.no/2022/a2a-fair-metrics/08-http-describedby-citeas-linkset-txt/\"", "rel=\"cite-as\""]
         href = elements.shift # first element is always the link url
         # warn "working on link href #{href}"
@@ -259,8 +269,6 @@ module LinkHeaders
         attrhash = {}
         elements.each do |e|
           key, val = e.split('=')
-          key.strip!
-          val.strip!
           val.delete_prefix!('"').delete_suffix!('"') # get rid of newlines and start/end quotes
           attrhash[key.to_sym] = val # split on key=val and make key a symbol
         end 

data/spec/linkheader/parser_spec.rb

@@ -12,26 +12,53 @@ factory = p.factory  # LinkHeaders::LinkFactory
 
 RSpec.describe LinkHeaders::Processor do
 
-  it 'has a version number' do
+  it 'Benchmark: has a version number' do
     expect(LinkHeaders::Processor::VERSION).not_to be nil
   end
 
-  it "should find PURL citeas which has described-by and cite-as in mixed HTTP and HTML headers" do
-    expect(factory.all_links.length).to eq 5
+  it "Benchmark: should find 8 links in total" do
+    expect(factory.all_links.length).to eq 8
   end
-  it "should find find href on all links" do
-    expect(factory.all_links.select{|l| l.href}.length).to eq 5
+  it "Benchmark: should find find href on all links" do
+    expect(factory.all_links.select{|l| l.href}.length).to eq 8
   end
-  it "should find find href on all links" do
-    expect(factory.all_links.select{|l| l.anchor}.length).to eq 5
+  it "Benchmark: should find find anchor on all links" do
+    expect(factory.all_links.select{|l| l.anchor}.length).to eq 8
   end
-  it "should find 5 links in mixed HTTP and HTML headers" do
-    expect(factory.all_links.select{|l| l.relation}.length).to eq 5
+  it "Benchmark: should find 5 links in mixed HTTP and HTML headers" do
+    expect(factory.all_links.select{|l| l.relation}.length).to eq 8
   end
-  it "should find one citeas in mixed HTTP and HTML headers" do
+  it "Benchmark: should find one citeas in mixed HTTP and HTML headers" do
     expect(factory.all_links.select{|l| l.relation == 'cite-as'}.length).to eq 1
   end
-  it "should find described-by in mixed HTTP and HTML headers" do
+  it "Benchmark: should find described-by in mixed HTTP and HTML headers" do
     expect(factory.all_links.select{|l| l.relation == 'describedby'}.length).to eq 1
   end
+
+  url2 = "https://doi.org/10.7910/DVN/Z2JD58"
+  p2 = LinkHeaders::Processor.new(default_anchor: url2)
+  r2 = RestClient.get(url2)
+  p2.extract_and_parse(response: r2)
+  factory2 = p2.factory  # LinkHeaders::LinkFactory
+  
+  it "Dataverse: should find 29 links in total" do
+    expect(factory2.all_links.length).to eq 28
+  end
+  it "Dataverse: should find find href on all links" do
+    expect(factory2.all_links.select{|l| l.href}.length).to eq 28
+  end
+  it "Dataverse: should find find anchor on all links" do
+    expect(factory2.all_links.select{|l| l.anchor}.length).to eq 28
+  end
+  it "Dataverse: should find one citeas" do
+    expect(factory2.all_links.select{|l| l.relation == 'cite-as'}.length).to eq 1
+  end
+  it "Dataverse: should find 2 described-by" do
+    expect(factory2.all_links.select{|l| l.relation == 'describedby'}.length).to eq 2
+  end
+  it "Dataverse: should find 1 license" do
+    expect(factory2.all_links.select{|l| l.relation == 'license'}.length).to eq 1
+  end
+
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: linkheaders-processor
 version: !ruby/object:Gem::Version
-  version: 0.1.17
+  version: 0.1.19
 platform: ruby
 authors:
 - Mark Wilkinson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-08-12 00:00:00.000000000 Z
+date: 2023-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -168,7 +168,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.28
+rubygems_version: 3.3.23
 signing_key: 
 specification_version: 4
 summary: A parser/processor for Link Headers and Linksets in both JSON and Text formats.