linkheaders-processor 0.1.8

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: af390c80d1304df2d885e4bb19ad6be8a95e695b7b88ac2a59762e8f11d17dff
+  data.tar.gz: f3e90daa90734be50afb722f6023ecb6594c778776c987c0a4bf9b42e4d3aeaa
+SHA512:
+  metadata.gz: 100903ef954dc3b40aaea1f97b285bb5dce59703a968905c9a9a7933416c1e4b847de32d0c03de5c646b3d0ae8d6d9a73ae003c315ee335aac10a956cfc38bfb
+  data.tar.gz: 71b0b8b7ad489ee6f3db7787fa6de0da3b3bbc40c2f16c53f16586be88233f3c88693c848624bd592d1a923ea9d90e535eaf6841547cade3e96664bdde6cdba4

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+## [Unreleased]
+
+## [0.1.0] - 2022-04-27
+
+- Initial release

data/Gemfile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in linkset-parser.gemspec
+gemspec
+
+gem "rake", "~> 13.0"
+
+gem "rspec", "~> 3.0"
+
+gem "rubocop", "~> 1.21"

data/Gemfile.lock

@@ -0,0 +1,150 @@
+PATH
+  remote: .
+  specs:
+    linkheaders-processor (0.1.8)
+      json (~> 2.0)
+      json-ld (~> 3.2)
+      json-ld-preloaded (~> 3.2)
+      metainspector (~> 5.11.2)
+      rest-client (~> 2.1)
+      securerandom (~> 0.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.8.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    ast (2.4.2)
+    diff-lcs (1.5.0)
+    domain_name (0.5.20190701)
+      unf (>= 0.0.5, < 1.0.0)
+    faraday (1.10.0)
+      faraday-em_http (~> 1.0)
+      faraday-em_synchrony (~> 1.0)
+      faraday-excon (~> 1.1)
+      faraday-httpclient (~> 1.0)
+      faraday-multipart (~> 1.0)
+      faraday-net_http (~> 1.0)
+      faraday-net_http_persistent (~> 1.0)
+      faraday-patron (~> 1.0)
+      faraday-rack (~> 1.0)
+      faraday-retry (~> 1.0)
+      ruby2_keywords (>= 0.0.4)
+    faraday-cookie_jar (0.0.7)
+      faraday (>= 0.8.0)
+      http-cookie (~> 1.0.0)
+    faraday-em_http (1.0.0)
+    faraday-em_synchrony (1.0.0)
+    faraday-encoding (0.0.5)
+      faraday
+    faraday-excon (1.1.0)
+    faraday-http-cache (2.4.0)
+      faraday (>= 0.8)
+    faraday-httpclient (1.0.1)
+    faraday-multipart (1.0.4)
+      multipart-post (~> 2)
+    faraday-net_http (1.0.1)
+    faraday-net_http_persistent (1.2.0)
+    faraday-patron (1.0.0)
+    faraday-rack (1.0.0)
+    faraday-retry (1.0.3)
+    faraday_middleware (1.2.0)
+      faraday (~> 1.0)
+    fastimage (2.2.6)
+    htmlentities (4.3.4)
+    http-accept (1.7.0)
+    http-cookie (1.0.5)
+      domain_name (~> 0.5)
+    json (2.6.2)
+    json-canonicalization (0.3.0)
+    json-ld (3.2.1)
+      htmlentities (~> 4.3)
+      json-canonicalization (~> 0.3)
+      link_header (~> 0.0, >= 0.0.8)
+      multi_json (~> 1.15)
+      rack (~> 2.2)
+      rdf (~> 3.2)
+    json-ld-preloaded (3.2.0)
+      json-ld (~> 3.2)
+      rdf (~> 3.2)
+    link_header (0.0.8)
+    metainspector (5.11.2)
+      addressable (~> 2.7)
+      faraday (~> 1.4)
+      faraday-cookie_jar (~> 0.0)
+      faraday-encoding (~> 0.0)
+      faraday-http-cache (~> 2.2)
+      faraday_middleware (~> 1.0)
+      fastimage (~> 2.2)
+      nesty (~> 1.0)
+      nokogiri (~> 1.11)
+    mime-types (3.4.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2022.0105)
+    multi_json (1.15.0)
+    multipart-post (2.2.3)
+    nesty (1.0.2)
+    netrc (0.11.0)
+    nokogiri (1.13.8-x86_64-linux)
+      racc (~> 1.4)
+    parallel (1.22.1)
+    parser (3.1.2.0)
+      ast (~> 2.4.1)
+    public_suffix (4.0.7)
+    racc (1.6.0)
+    rack (2.2.4)
+    rainbow (3.1.1)
+    rake (13.0.6)
+    rdf (3.2.8)
+      link_header (~> 0.0, >= 0.0.8)
+    regexp_parser (2.5.0)
+    rest-client (2.1.0)
+      http-accept (>= 1.7.0, < 2.0)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 4.0)
+      netrc (~> 0.8)
+    rexml (3.2.5)
+    rspec (3.11.0)
+      rspec-core (~> 3.11.0)
+      rspec-expectations (~> 3.11.0)
+      rspec-mocks (~> 3.11.0)
+    rspec-core (3.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-expectations (3.11.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.11.0)
+    rspec-mocks (3.11.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.11.0)
+    rspec-support (3.11.0)
+    rubocop (1.32.0)
+      json (~> 2.3)
+      parallel (~> 1.10)
+      parser (>= 3.1.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.19.1, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.19.1)
+      parser (>= 3.1.1.0)
+    ruby-progressbar (1.11.0)
+    ruby2_keywords (0.0.5)
+    securerandom (0.1.1)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.8.2)
+    unicode-display_width (2.2.0)
+
+PLATFORMS
+  x86_64-linux
+
+DEPENDENCIES
+  linkheaders-processor!
+  rake (~> 13.0)
+  rspec (~> 3.0)
+  rubocop (~> 1.21)
+
+BUNDLED WITH
+   2.3.12

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Mark Wilkinson
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,77 @@
+# LinkHeader::Parser
+
+A gem to extract Link Headers from Web responses.
+
+This module handles HTTP Link Headers, HTML Link Headers, and auto-follows links to LinkSets in both JSON and Text format, and processes them also.
+
+## Installation
+
+Install the gem and add to the application's Gemfile by executing:
+
+    $ bundle add linkheader-processor
+
+If bundler is not being used to manage dependencies, install the gem by executing:
+
+    $ gem install linkheader-processor
+
+## Usage
+
+
+```
+
+    require 'linkheader/processor'
+    require 'rest-client'
+
+    # url1 has http link headers, and a reference to a linkset in json format
+    url1 = "https://s11.no/2022/a2a-fair-metrics/07-http-describedby-citeas-linkset-json/"
+
+    # url2 has http link headers, with a reference to a linkset in legacy text format
+    url2 = "https://s11.no/2022/a2a-fair-metrics/28-http-linkset-txt-only/"
+
+    p = LinkHeader::Parser.new(default_anchor: url1)
+    r = RestClient.get(url1)
+
+    p.extract_and_parse(response: r)
+    factory = p.factory  # LinkHeader::LinkFactory
+
+    factory.all_links.each do |l| 
+        puts l.href
+        puts l.relation
+        puts l.responsepart
+
+        puts l.linkmethods  # returns list of instance methods beyond href and relation, that are attributes of the link
+        l.linkmethods.each do |method|
+            puts "#{method}=" + l.send(method)
+        end
+        puts
+    end
+
+
+
+    p = LinkHeader::Parser.new(default_anchor: url2)
+    r = RestClient.get(url2)
+
+    p.extract_and_parse(response: r)
+    factory = p.factory
+
+    factory.all_links.each do |l| 
+        puts l.href
+        puts l.relation
+        puts l.responsepart
+        puts
+        puts
+    end
+
+```
+
+
+## Development
+
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/markwilkinson/linkheader-parser.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+task default: %i[spec rubocop]

data/launch.json

@@ -0,0 +1,11 @@
+{
+    "name": "RSpec - all",
+    "type": "Ruby",
+    "request": "launch",
+    "cwd": "${workspaceRoot}",
+    "program": "/home/osboxes/.rvm/gems/ruby-3.0.0/bin/rspec",
+    "args": [
+      "-I",
+      "${workspaceRoot}"
+    ]
+  }

data/lib/linkheaders/constants.rb

@@ -0,0 +1,28 @@
+ACCEPT_ALL_HEADER = {'Accept' => 'text/turtle, application/ld+json, application/rdf+xml, text/xhtml+xml, application/n3, application/rdf+n3, application/turtle, application/x-turtle, text/n3, text/turtle, text/rdf+n3, text/rdf+turtle, application/n-triples' }
+
+TEXT_FORMATS = {
+    'text' => ['text/plain',],
+}
+
+RDF_FORMATS = {
+  'jsonld'  => ['application/ld+json', 'application/vnd.schemaorg.ld+json'],  # NEW FOR DATACITE
+  'turtle'  => ['text/turtle','application/n3','application/rdf+n3',
+               'application/turtle', 'application/x-turtle','text/n3','text/turtle',
+               'text/rdf+n3', 'text/rdf+turtle'],
+  #'rdfa'    => ['text/xhtml+xml', 'application/xhtml+xml'],
+  'rdfxml'  => ['application/rdf+xml'],
+  'triples' => ['application/n-triples','application/n-quads', 'application/trig']
+}
+
+XML_FORMATS = {
+  'xml' => ['text/xhtml','text/xml',]
+}
+
+HTML_FORMATS = {
+  'html' => ['text/html','text/xhtml+xml', 'application/xhtml+xml']
+}
+
+JSON_FORMATS = {
+            'json' => ['application/json',]
+}
+

data/lib/linkheaders/link.rb

@@ -0,0 +1,187 @@
+module LinkHeaders
+  class LinkFactory
+
+    # @return [<String>] the HTTP anchor used by default for implicit Links
+    attr_accessor :default_anchor
+    # @return [Array] An array of strings containing any warnings that were encountered when creating the link (e.g. duplicate cite-as but non-identical URLs)
+    attr_accessor :warnings
+    @@all_links = Array.new
+
+    #
+    # Create the LinkFacgtory Object
+    #
+    # @param [String] default_anchor The URL to be used as the default anchor for a link when it isn't specified
+    #
+    def initialize(default_anchor: 'https://example.org/')
+      @default_anchor = default_anchor
+      @warnings = Array.new
+    end
+
+    #
+    # Create a new LinkHeader::Link object
+    #
+    # @param [Symbol] responsepart either :header, :body, or :linkset as the original location of this Link
+    # @param [String] href the URL of the link
+    # @param [String] relation the string of the relation type (e.g. "cite-as" or "described-by")
+    # @param [String] anchor The URL of the anchor.  Defaults to the default anchor of the LinkHeader factory
+    # @param [Hash] **kwargs All other facets of the link. e.g. 'type' => 'text/html',...
+    #
+    # @return [LinkHeader::Link] The Link object just created
+    #
+    def new_link(responsepart:, href:, relation:, anchor: @default_anchor, **kwargs)
+      # warn "creating new link with kw #{kwargs}"
+      link = LinkHeader::Link.new(responsepart: responsepart, factory: self, href: href, anchor: anchor, relation: relation, **kwargs)
+      link = sanitycheck(link)  # this will add warnings if the link already exists and has a conflict.  returns the original of a duplicate
+      @@all_links |= [link]
+      return link
+    end
+
+    #
+    # retrieve all known LinkHeader::Link objects
+    #
+    # @return [Array] Array of all LinkHeader::Link objects created by the factory so far
+    #
+    def all_links
+      @@all_links
+    end
+
+    #
+    # Extracts Linkset type links from a list of LinkHeader::Link objects
+    #
+    # @return [Array] Array of LinkHeader::Link objects that represent URLs of LinkSets.
+    #
+    def linksets
+      links = Array.new
+      self.all_links.each do |link|
+        # warn "found #{link.relation}"
+        next unless link.relation == 'linkset'
+        links << link
+      end
+     links
+    end
+
+    #
+    # Extracts the LinkHeader::Link ojects that originated in the HTTP Headers
+    #
+    # @return [Array]  Array of LinkHeader::Link objects 
+    #
+    def headlinks
+      links = Array.new
+      self.all_links.each do |link|
+        # warn "found #{link.relation}"
+        next unless link.responsepart == :header
+        links << link
+      end
+      links
+    end
+
+    #
+    # Extracts the LinkHeader::Link ojects that originated in the HTML Link Headers
+    #
+    # @return [Array]  Array of LinkHeader::Link objects
+    #
+    def bodylinks
+      links = Array.new
+      self.all_links.each do |link|
+        # warn "found #{link.relation}"
+        next unless link.responsepart == :body
+        links << link
+      end
+      links
+    end
+
+    #
+    # Extracts the LinkHeader::Link ojects that originated from a LinkSet
+    #
+    # @return [Array]  Array of LinkHeader::Link objects
+    #
+    def linksetlinks
+      links = Array.new
+      self.all_links.each do |link|
+        # warn "found #{link.relation}"
+        next unless link.responsepart == :linkset
+        links << link
+      end
+      links
+    end
+
+    def sanitycheck(link)
+      flag = true
+      self.all_links.each do |l|
+        if l.relation == "cite-as" and link.relation == "cite-as"
+          if l.href != link.href
+            @warnings << 'WARN: Found conflicting cite-as relations.  This should never happen'
+          end
+        end
+        if l.href == link.href
+          if l.relation != link.relation
+            @warnings << 'WARN: Found identical hrefs with different relation types.  This may be suspicious. Both have been retained'
+          end
+          if l.relation = link.relation
+            @warnings << 'WARN: found apparent duplicate. Ignoring and returning known link'
+            link = l
+          end
+        end
+      end
+      link
+    end
+  end
+
+  #
+  # LinkHeader::Link represnts an HTTP Link Header, an HTML LinkHeader, or a LinkSet Link.
+  #
+  # #anchor, #href, and #relation are all guaranteed to return a value.  Other methods are dynamically created based on what key/value pairs exist in the link
+  # for example, if "'type': 'text/html'" exists in the link description, then the method #type will be available on the Link object.
+  #
+  class Link
+    # @return [String] URL of the Link anchor
+    attr_accessor :anchor 
+    # @return [String] URL of the Link
+    attr_accessor :href
+    # @return [String] What is the relation? (e.g. "cite-as")
+    attr_accessor :relation
+    # @return [LinkHeader::LinkFactory] The factory that made the Link
+    attr_accessor :factory
+    # @return [Symbol] :header, :body, or :linkset indicating the place the Link object originated
+    attr_accessor :responsepart
+    # @return [String] the list of instance method names auto-generated by the various key/value pairs in the link header.  e.g. "type"
+    attr_accessor :linkmethods
+    
+
+    #
+    # Create the Link object
+    #
+    # @param [Symbol] responsepart :header, :body, :linkset
+    # @param [LinkHeader::LinkFactory] factory the factory that made the link
+    # @param [String] href The URL of the Link
+    # @param [String] anchor The URL of the anchor
+    # @param [String] relation the Link relation (e.g. "cite-as")
+    # @param [hash] **kwargs The remaining facets of the link (e.g. type => 'text/html')
+    #
+    def initialize(responsepart:, factory:, href:, anchor:, relation:, **kwargs)
+      # warn "incoming kw args #{kwargs}"
+      @href = href
+      @anchor = anchor
+      @relation = relation
+      @factory = factory
+      @responsepart = responsepart
+      @linkmethods = Array.new
+
+      kwargs.each do |k, v|
+        # warn "key #{k} val #{v}"
+
+        @linkmethods << k
+        define_singleton_method(k.to_sym) {
+          value = instance_variable_get("@#{k}")
+          return value
+        } 
+        define_singleton_method "#{k}=".to_sym do |val|
+          instance_variable_set("@#{k}", val)
+          return "@#{k}".to_sym
+        end
+        # warn "methods:  #{self.methods - Object.new.methods}"
+        self.send("#{k}=", v)
+      end
+    end
+  end
+end

data/lib/linkheaders/processor/version.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+
+module LinkHeaders
+  class Processor
+    VERSION = "0.1.8"
+  end
+end

data/lib/linkheaders/processor.rb

@@ -0,0 +1,250 @@
+# frozen_string_literal: true
+
+require_relative 'processor/version'
+require_relative 'constants'
+require_relative 'link'
+require_relative 'web_utils'
+
+require 'json'
+require 'rest-client'
+require 'securerandom'
+require 'metainspector'
+
+module LinkHeaders
+  class Error < StandardError; end
+
+  # A Link Header parser
+  #
+  # Works for both HTML and HTTP links, and handles references to Linksets of either JSON or Text types
+  #
+  class Parser
+    # @return [<Type>] <description>
+    attr_accessor :default_anchor, :factory
+
+    #
+    # Create the Link Headers Parser and its Link factory
+    #
+    # @param [<String>] default_anchor Link relations always have an anchor, but it is sometimes implicit.  This value will be used in implicit cases.
+    #
+    def initialize(default_anchor: 'https://default.anchor.org/')
+      @default_anchor = default_anchor
+      @factory = LinkHeader::LinkFactory.new(default_anchor: @default_anchor)
+    end
+
+    #
+    # Get the parser factory that contains all the links
+    #
+    # @return [<LinkHeader::LinkFactory>] The factory containing the links (LinkHeader::Link) that have been created so far
+    #
+    def factory
+      @factory
+    end
+
+    #
+    # Parses a RestClient::Response
+    #
+    # The HTTP headers are parsed for Links and if those links contain a Linkset, that is retrieved and parsed
+    # If the Response is of some HTML form, this is also parsed for Link headers and Linkset links
+    # All discovered links end up in a LinkHeader::LinkFactory object (self.factory)
+    #
+    # @param [<RestClilent::Response>] response The full response object from an HTTP 2** successful call
+    #
+    #
+    def extract_and_parse(response: RestClient::Response.new)
+      head = response.headers
+      body = response.body
+      # warn "\n\n head #{head.inspect}\n\n"
+
+      unless head
+        warn "WARNING: This doesn't seem to be a RestClient response message.\nReturning blank"
+        return [[], []]
+      end
+
+      parse_http_link_headers(head) # pass guid to check against anchors in linksets
+      HTML_FORMATS['html'].each do |format|
+        if head[:content_type] and head[:content_type].match(format)
+          htmllinks = parse_html_link_headers(body) # pass html body to find HTML link headers
+        end
+      end
+    end
+
+    #
+    # Consume a String of the Link Headers and parse it into individual links. Will automatically retrieve and process any LinkSet references found.  All LinkHeader::Link objects end up in the LinkHeader::LinkFactory object (self.factory)
+    #
+    # @param [RestClient::Response::Header] headers  the Headers of a RestClent::Response.  Calls headers[:link] to retrieve '<https://example.one.com>; rel="preconnect", <https://example.two.com>; rel="preconnect",  <https://example.three.com>; rel="preconnect"'
+    #
+    #
+    def parse_http_link_headers(headers)
+
+      # Link: <https://example.one.com>; rel="preconnect", <https://example.two.com>; rel="preconnect",  <https://example.three.com>; rel="preconnect"
+      links = headers[:link]
+      return [] unless links
+
+      # warn links.inspect
+      parts = links.split(',') # ["<https://example.one.com>; rel='preconnect'", "<https://example.two.com>; rel="preconnect"".....]
+      # warn parts
+
+      # Parse each part into a named link
+      split_http_link_headers(parts) # creates links from the split headers and adds to factory.all_links
+      check_for_linkset(responsepart: :header)  # all links are held in the Linkset::LinkFactory object (factory variable here).  This scans the links for a linkset link to follow
+    end
+
+    def split_http_link_headers(parts)
+      parts.each do |part, _index|
+        # warn "link is:  #{part}"
+
+        section = part.split(';') # ["<https://example.one.com>", "rel='preconnect'"]
+        # warn section
+        next unless section[0]
+
+        href = section[0][/<(.*)>/, 1]
+        next unless section[1]
+
+        sections = {}
+        section[1..].each do |s| # can be more than one link property "rel='preconnect'"
+          s.strip!
+          unless m = s.match(%r{(\w+?)="?([\w:\d.,\#\-+/\s]+)"?})
+            next
+          end # can be rel="cite-as describedby"  --> two relations in one!  or "linkset+json"
+
+          relation = m[1] # rel"
+          value = m[2] # "preconnect"
+          sections[relation] = value # value could hold multiple relation types   sections[:rel] = "preconnect"
+        end
+        next unless sections['rel']  # the relation is required!
+
+        anchor = sections['anchor'] || default_anchor
+        sections.delete('anchor')
+        relation = sections['rel']
+        sections.delete('rel')
+
+        factory.new_link(responsepart: :header, anchor: anchor, href: href, relation: relation, **sections) # parsed['https://example.one.com'][:rel] = "preconnect"
+      end
+    end
+
+    #
+    # Parses the link headers out of an HTML body, and adds links to the LinkHeader::LinkFactory object.  Will automatically retrieve and process any LinkSet references found
+    #
+    # @param [String] body The HTML of the page containing HTML Link headers
+    #
+    def parse_html_link_headers(body)
+      m = MetaInspector.new('http://example.org', document: body)
+      # an array of elements that look like this: [{:rel=>"alternate", :type=>"application/ld+json", :href=>"http://scidata.vitk.lv/dataset/303.jsonld"}]
+
+      m.head_links.each do |l|
+        # warn "link is:  #{l}"
+        next unless l[:href] and l[:rel] # required
+
+        anchor = l[:anchor] || default_anchor
+        l.delete(:anchor)
+        relation = l[:rel]
+        l.delete(:rel)
+        href = l[:href]
+        l.delete(:href)
+        factory.new_link(responsepart: :body, anchor: anchor, href: href, relation: relation, **l)
+      end
+      check_for_linkset(responsepart: :body)
+    end
+
+    def check_for_linkset(responsepart:)
+      # warn "looking for a linkset"
+      factory.linksets.each do |linkset|
+        # warn "found #{linkset.methods- Object.new.methods}"
+        # warn "inspect #{linkset.inspect}"
+        next unless linkset.respond_to? 'type'
+        # warn "responds #{linkset.type}  "
+        case linkset.type
+        when 'application/linkset+json'
+          # warn "found a json linkset"
+          processJSONLinkset(href: linkset.href)
+        when 'application/linkset'
+          # warn "found a text linkset"
+          processTextLinkset(href:linkset.href)
+        else
+          warn "the linkset #{linkset} was not typed as 'application/linkset+json' or 'application/linkset', and it should be! (found #{linkset.type}) Ignoring..."
+        end
+      end
+    end
+
+    def processJSONLinkset(href:)
+      _headers, linkset = fetch(href, { 'Accept' => 'application/linkset+json' })
+      # warn "Linkset body #{linkset.inspect}"
+
+      return nil unless linkset
+
+      # linkset = '{ "linkset":
+      #   [
+      #     { "anchor": "http://example.net/bar",
+      #       "item": [
+      #         {"href": "http://example.com/foo1", "type": "text/html"},
+      #         {"href": "http://example.com/foo2"}
+      #       ],
+      #       "next": [
+      #         {"href": "http://the.next/"}
+      #       ]
+      #     }
+      #   ]
+      # }'
+
+      linkset = JSON.parse(linkset)
+      linkset['linkset'].each do |ls|
+        # warn ls.inspect, "\n"
+        anchor = ls['anchor'] || @default_anchor
+        ls.delete('anchor') if ls['anchor'] # we need to delete since all others have a list as a value
+        attrhash = {}
+        # warn ls.keys, "\n"
+
+        ls.each_key do |reltype| # key =  e.g. "item", "described-by". "cite"
+          # warn reltype, "\n"
+          # warn ls[reltype], "\n"
+          ls[reltype].each do |attrs|  # attr = e.g.  {"href": "http://example.com/foo1", "type": "text/html"}
+            next unless attrs['href']  # this is a required attribute of a  linkset relation
+
+            href = attrs['href']
+            # now go through the other attributes of that relation
+            attrs.each do |attr, val| # attr = e.g. "type"; val = "text/html"
+              attrhash[attr.to_sym] = val
+            end
+          end
+          factory.new_link(responsepart: :linkset, href: href, relation: reltype, anchor: anchor, **attrhash)
+        end
+      end
+    end
+
+    def processTextLinkset(href:)
+      headers, linkset = fetch(href, { 'Accept' => 'application/linkset' })
+      # warn "linkset body #{linkset.inspect}"
+      return {} unless linkset
+
+      links = linkset.scan(/(<.*?>[^<]+)/) # split on the open angle bracket, which indicates a new link
+      # warn "Links found #{links}"
+
+      links.each do |ls|
+        # warn "workking on link #{ls}"
+        ls = ls.first # ls is a single element array
+        elements = ls.split(';') # semicolon delimited fields
+        # ["<https://w3id.org/a2a-fair-metrics/08-http-describedby-citeas-linkset-txt/>", "anchor=\"https://s11.no/2022/a2a-fair-metrics/08-http-describedby-citeas-linkset-txt/\"", "rel=\"cite-as\""]
+        href = elements.shift # first element is always the link url
+        # warn "working on link href #{href}"
+        href = href.match(/<([^>]+)>/)[1]
+        attrhash = {}
+        elements.each do |e|
+          key, val = e.split('=')
+          key.strip!
+          val.strip!
+          val.delete_prefix!('"').delete_suffix!('"') # get rid of newlines and start/end quotes
+          attrhash[key.to_sym] = val # split on key=val and make key a symbol
+        end 
+        warn "No link relation type... this is bad!  Skipping" unless attrhash[:rel]
+        next unless attrhash[:rel]
+        reltype = attrhash[:rel]
+        attrhash.delete(:rel)
+        anchor = attrhash[:anchor] || @default_anchor
+        attrhash.delete(:anchor)
+        
+        factory.new_link(responsepart: :linkset, href: href, relation: reltype, anchor: anchor, **attrhash)
+        # warn "created #{[href, reltype, anchor, **attrhash]}"
+      end
+    end
+  end
+end

data/lib/linkheaders/web_utils.rb

@@ -0,0 +1,39 @@
+def fetch(url, headers = ACCEPT_ALL_HEADER)  # we will try to retrieve turtle whenever possible
+  # warn "In fetch routine now.  "
+
+  # warn "executing call over the Web to #{url.to_s}"
+  response = RestClient::Request.execute({
+                                           method: :get,
+                                           url: url.to_s,
+                                           # user: user,
+                                           # password: pass,
+                                           headers: headers
+                                         })
+
+  # warn "There was a response to the call #{url.to_s}"
+  # warn "Response code #{response.code}"
+  if response.code == 203
+    warn "WARN: Response is non-authoritative (HTTP response code: #{response.code}).  Headers may have been manipulated encountered when trying to resolve #{url}\n"
+  end
+  [response.headers, response.body]
+rescue RestClient::ExceptionWithResponse => e
+  warn "EXCEPTION WITH RESPONSE! #{e.response}\n#{e.response.headers}"
+  warn "WARN: HTTP error #{e} encountered when trying to resolve #{url}\n"
+  if e.response.code == 500
+    [false, false]
+  else
+    [e.response.headers, e.response.body]
+  end
+# now we are returning the headers and body that were returned
+rescue RestClient::Exception => e
+  warn "EXCEPTION WITH NO RESPONSE! #{e}"
+  warn "WARN: HTTP error #{e} encountered when trying to resolve #{url}\n"
+  [false, false]
+# now we are returning 'False', and we will check that with an \"if\" statement in our main code
+rescue Exception => e
+  warn "EXCEPTION UNKNOWN! #{e}"
+  warn "WARN: HTTP error #{e} encountered when trying to resolve #{url}\n"
+  [false, false]
+  # now we are returning 'False', and we will check that with an \"if\" statement in our main code
+  # you can capture the Exception and do something useful with it!\n",
+end

data/sig/linkheader/parser.rbs

@@ -0,0 +1,6 @@
+module Linkset
+  module Parser
+    VERSION: String
+    # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+  end
+end

data/spec/linkheader/parser_spec.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+RSpec.describe LinkHeader::Parser do
+  it 'has a version number' do
+    expect(LinkHeader::Parser::VERSION).not_to be nil
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "linkset/parser"
+
+RSpec.configure do |config|
+  # Enable flags like --only-failures and --next-failure
+  config.example_status_persistence_file_path = ".rspec_status"
+
+  # Disable RSpec exposing methods globally on `Module` and `main`
+  config.disable_monkey_patching!
+
+  config.expect_with :rspec do |c|
+    c.syntax = :expect
+  end
+end

data/testme.rb

@@ -0,0 +1,37 @@
+require 'linkheader/processor'
+#require_relative 'lib/linkheader/processor'
+require 'rest-client'
+
+url1 = "https://s11.no/2022/a2a-fair-metrics/07-http-describedby-citeas-linkset-json/"
+url2 = "https://s11.no/2022/a2a-fair-metrics/28-http-linkset-txt-only/"
+
+p = LinkHeader::Parser.new(default_anchor: url1)
+r = RestClient.get(url1)
+
+p.extract_and_parse(response: r)
+factory = p.factory
+
+factory.all_links.each do |l| 
+    puts l.href
+    puts l.relation
+    puts l.responsepart
+    puts
+    puts
+end
+
+
+
+p = LinkHeader::Parser.new(default_anchor: url2)
+r = RestClient.get(url2)
+
+p.extract_and_parse(response: r)
+factory = p.factory
+
+factory.all_links.each do |l| 
+    puts l.href
+    puts l.relation
+    puts l.responsepart
+    puts
+    puts
+end
+

metadata

@@ -0,0 +1,164 @@
+--- !ruby/object:Gem::Specification
+name: linkheaders-processor
+version: !ruby/object:Gem::Version
+  version: 0.1.8
+platform: ruby
+authors:
+- Mark Wilkinson
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2022-07-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: json-ld
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: json-ld-preloaded
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: securerandom
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+- !ruby/object:Gem::Dependency
+  name: metainspector
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.11.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.11.2
+description: A parser/processor for Link Headers and Linksets in both JSON and Text
+  formats.
+email:
+- markw@illuminae.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- launch.json
+- lib/linkheaders/constants.rb
+- lib/linkheaders/link.rb
+- lib/linkheaders/processor.rb
+- lib/linkheaders/processor/version.rb
+- lib/linkheaders/web_utils.rb
+- sig/linkheader/parser.rbs
+- spec/linkheader/parser_spec.rb
+- spec/spec_helper.rb
+- testme.rb
+homepage: https://github.com/markwilkinson/linkheader-processor
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/markwilkinson/linkheader-processor
+  source_code_uri: https://github.com/markwilkinson/linkheader-processor
+  changelog_uri: https://github.com/markwilkinson/linkheader-processor/blob/master/CHANGELOG.md
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.28
+signing_key: 
+specification_version: 4
+summary: A parser/processor for Link Headers and Linksets in both JSON and Text formats.
+test_files:
+- spec/linkheader/parser_spec.rb
+- spec/spec_helper.rb