relaton-nist 1.9.1 → 1.9.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d440504b749834c27875dc8a1ee84346451578db5dd1588b75eafdb31087b64c
-  data.tar.gz: ecfb305d172a8afa171fa4493b1ffec2dd458fd413143a7f197034a392ddf451
+  metadata.gz: c8d36c40a2b5307192f8d25cc103cbb5f08e0b21a0a88940c8e7688417c01f4d
+  data.tar.gz: '048f67e6cb6835f382973576d8c97048a7462a512d7b5f8469bed3af07ec594f'
 SHA512:
-  metadata.gz: 7af96feb236bddbabd7e90982637b608813d5c2163c7453f217c658035cf34b0ef80adc6441d6be213b77e62b5625d95fb10a30914fc94a1b6cf966b1c921bf6
-  data.tar.gz: 9b9daa02fc2ec7df33ad5c8cdf768c711b1bac0122191e7cfc79ee7c8ed2b7de00fbe6509b3347f387cf4868b24448f91f1a687e45b80fe65ce31d3f7a46cad5
+  metadata.gz: e70ce8ef01f291b254e095c33ba639cfc8fcf23736357b1eed465498fc989f101fe1d05fc031726498b917f6bd227a6a3d7e1171d18ef455f693452da48a3495
+  data.tar.gz: f9e38480b0c3d73d48485988f86350cd50378489814328622cd7cf6da592056870ca74779feae53a607f737aa4fa2f5f8ae1323450f358d782133d9f48b6c6a1

data/.gitignore

@@ -7,6 +7,7 @@
 /spec/reports/
 /tmp/
 .vscode/
+/data/
 .rubocop-https---raw-githubusercontent-com-riboseinc-oss-guides-master-ci-rubocop-yml
 
 # rspec failure tracking

data/.rubocop.yml

@@ -2,6 +2,8 @@
 # https://github.com/riboseinc/oss-guides
 # All project-specific additions and overrides should be specified in this file.
 
+require: rubocop-rails
+
 inherit_from:
   - https://raw.githubusercontent.com/riboseinc/oss-guides/master/ci/rubocop.yml
 AllCops:

data/README.adoc

@@ -147,6 +147,17 @@ item.docidentifier.first.id
 => "SP 800-38A-Add"
 ----
 
+=== Typed links
+
+NIST documents may have `src` and `doi` link types.
+
+[source,ruby]
+----
+item.link
+=> [#<RelatonBib::TypedUri:0x00007f901971dc10 @content=#<Addressable::URI:0x62c URI:https://csrc.nist.gov/publications/detail/sp/800-67/rev-2/final>, @type="src">,
+ #<RelatonBib::TypedUri:0x00007f901971d6e8 @content=#<Addressable::URI:0x640 URI:https://doi.org/10.6028/NIST.SP.800-67r2>, @type="doi">]
+----
+
 === Create bibliographic item from YAML
 [source,ruby]
 ----
@@ -159,6 +170,25 @@ RelatonNist::NistBibliographicItem.from_hash hash
 ...
 ----
 
+=== Fetch data
+
+This gem uses the https://raw.githubusercontent.com/usnistgov/NIST-Tech-Pubs/nist-pages/xml/allrecords.xml dataset as one of data sources.
+
+The method `RelatonNist::DataFetcher.fetch(output: "data", format: "yaml")` fetches all the documents from the datast and save them to the `./data` folder in YAML format.
+Arguments:
+
+- `output` - folder to save documents (default './data').
+- `format` - format in which the documents are saved. Possimle formats are: `yaml`, `xml`, `bibxxml` (default `yaml`).
+
+[source,ruby]
+----
+RelatonNist::DataFetcher.fetch
+Started at: 2021-09-01 18:01:01 +0200
+Stopped at: 2021-09-01 18:01:43 +0200
+Done in: 42 sec.
+=> nil
+----
+
 == Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/relaton_nist/data_fetcher.rb

@@ -0,0 +1,255 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module RelatonNist
+  class DataFetcher
+    RELATION_TYPES = {
+      "replaces" => "obsoletes",
+      "isVersionOf" => "editionOf",
+      "hasTranslation" => "hasTranslation",
+      "isTranslationOf" => "translatedFrom",
+      "hasPreprint" => "hasReprint",
+      "isSupplementTo" => "complements",
+      "isPartOf" => "partOf",
+      "hasPart" => "hasPart",
+    }.freeze
+    URL = "https://raw.githubusercontent.com/usnistgov/NIST-Tech-Pubs/nist-pages/xml/allrecords.xml"
+
+    def initialize(output, format)
+      @output = output
+      @format = format
+      @ext = format.sub(/^bib/, "")
+    end
+
+    def parse_docid(doc) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      # case doi
+      # when "10.6028/NBS.CIRC.12e2revjune" then doi.sub!("13e", "12e")
+      # when "10.6028/NBS.CIRC.36e2" then doi.sub!("46e", "36e")
+      # when "10.6028/NBS.HB.67suppJune1967" then doi.sub!("1965", "1967")
+      # when "10.6028/NBS.HB.105-1r1990" then doi.sub!("105-1-1990", "105-1r1990")
+      # when "10.6028/NIST.HB.150-10-1995" then doi.sub!(/150-10$/, "150-10-1995")
+      # end
+      # anchor = doi.split("/")[1..-1].join "/"
+      [
+        { type: "NIST", id: pub_id(doc) },
+        { type: "DOI", id: doi(doc) },
+        { type: "NIST", id: anchor(doc), scope: "anchor" },
+      ]
+    end
+
+    def pub_id(doc)
+      anchor(doc).gsub(".", " ")
+    end
+
+    def doi(doc)
+      doc.at("doi_data/doi").text
+    end
+
+    def anchor(doc)
+      doi(doc).split("/")[1..-1].join "/"
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<RelatonBib::DocumentIdentifier>]
+    def fetch_docid(doc)
+      parse_docid(doc).map do |id|
+        RelatonBib::DocumentIdentifier.new(type: id[:type], id: id[:id])
+      end
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [RelatonBib::TypedTitleStringCollection, Array]
+    def fetch_title(doc)
+      t = doc.xpath("titles/title|titles/subtitle")
+      return [] unless t.any?
+
+      RelatonBib::TypedTitleString.from_string t.map(&:text).join, "en", "Latn"
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<RelatonBib::BibliographicDate>]
+    def fetch_date(doc)
+      doc.xpath("publication_date|approval_date").map do |dt|
+        on = dt.at("year").text
+        if (m = dt.at "month")
+          on += "-#{m.text}"
+          d = dt.at "day"
+          on += "-#{d.text}" if d
+        end
+        type = dt.name == "publication_date" ? "published" : "confirmed"
+        RelatonBib::BibliographicDate.new(type: type, on: on)
+      end
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [String]
+    def fetch_edition(doc)
+      doc.at("edition_number")&.text
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<Hash>]
+    def fetch_relation(doc)
+      ns = "http://www.crossref.org/relations.xsd"
+      doc.xpath("./ns:program/ns:related_item", ns: ns).map do |rel|
+        rdoi = rel.at_xpath("ns:intra_work_relation|ns:inter_work_relation", ns: ns)
+        fref = RelatonBib::FormattedRef.new content: rdoi.text
+        bibitem = RelatonBib::BibliographicItem.new formattedref: fref
+        type = RELATION_TYPES[rdoi["relationship-type"]]
+        warn "Relation type #{rdoi['relationship-type']} not found" unless type
+        { type: type, bibitem: bibitem }
+      end
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<RelatonBib::TypedUri>]
+    def fetch_link(doc)
+      url = doc.at("doi_data/resource").text
+      [RelatonBib::TypedUri.new(type: "doi", content: url)]
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<RelatonBib::FormattedString>]
+    def fetch_abstract(doc)
+      doc.xpath("jats:abstract/jats:p", "jats" => "http://www.ncbi.nlm.nih.gov/JATS1").map do |a|
+        RelatonBib::FormattedString.new(content: a.text, language: doc["language"], script: "Latn")
+      end
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<Hash>]
+    def fetch_contributor(doc) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+      contribs = doc.xpath("contributors/person_name").map do |p|
+        forename = []
+        initial = []
+        p.at("given_name")&.text&.split&.each do |fn|
+          if /^(?<init>\w)\.?$/ =~ fn
+            initial << RelatonBib::LocalizedString.new(init, doc["language"], "Latn")
+          else
+            forename << RelatonBib::LocalizedString.new(fn, doc["language"], "Latn")
+          end
+        end
+        sname = p.at("surname").text
+        surname = RelatonBib::LocalizedString.new sname, doc["language"], "Latn"
+        initial = []
+        ident = p.xpath("ORCID").map do |id|
+          RelatonBib::PersonIdentifier.new "orcid", id.text
+        end
+        fullname = RelatonBib::FullName.new(
+          surname: surname, forename: forename, initial: initial, identifier: ident,
+        )
+        person = RelatonBib::Person.new name: fullname, affiliation: affiliation(doc)
+        { entity: person, role: [{ type: p["contributor_role"] }] }
+      end
+      contribs + doc.xpath("publisher").map do |p|
+        abbr = p.at("../institution/institution_acronym")&.text
+        place = p.at("./publisher_place")
+        cont = []
+        if place
+          city, state = place.text.split(", ")
+          cont << RelatonBib::Address.new(street: [], city: city, state: state, country: "US")
+        end
+        org = RelatonBib::Organization.new(
+          name: p.at("publisher_name").text, abbreviation: abbr, contact: cont,
+        )
+        { entity: org, role: [{ type: "publisher" }] }
+      end
+    end
+
+    def affiliation(doc)
+      doc.xpath("./institution/institution_department").map do |id|
+        org = RelatonBib::Organization.new name: id.text
+        RelatonBib::Affiliation.new organization: org
+      end
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<String>]
+    def fetch_place(doc)
+      doc.xpath("institution/institution_place").map(&:text)
+    end
+
+    def fetch_series(doc)
+      title = RelatonBib::TypedTitleString.new(content: "NIST")
+      [RelatonBib::Series.new(title: title, number: pub_id(doc))]
+    end
+
+    #
+    # Save document
+    #
+    # @param bib [RelatonNist::NistBibliographicItem]
+    #
+    def write_file(bib) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      id = bib.docidentifier[0].id.gsub(%r{[/\s:.]}, "_").upcase.sub(/^NIST_IR/, "NISTIR")
+      file = File.join(@output, "#{id}.#{@ext}")
+      if File.exist? file
+        warn "File #{file} exists. Docid: #{bib.docidentifier[0].id}"
+        # warn "Link: #{bib.link.detect { |l| l.type == 'src' }.content}"
+      else
+        output = case @format
+                 when "yaml" then bib.to_hash.to_yaml
+                 when "xml" then bib.to_xml bibdata: true
+                 else bib.send "to_#{@format}"
+                 end
+        File.write file, output, encoding: "UTF-8"
+      end
+    end
+
+    #
+    # Create a document instance an save it.
+    #
+    # @param doc [Nokogiri::XML::Element]
+    #
+    # @raise [StandardError]
+    #
+    def parse_doc(doc) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
+      # mtd = doc.at('doi_record/report-paper/report-paper_metadata')
+      item = RelatonNist::NistBibliographicItem.new(
+        type: "standard", docid: fetch_docid(doc), title: fetch_title(doc),
+        link: fetch_link(doc), abstract: fetch_abstract(doc),
+        date: fetch_date(doc), edition: fetch_edition(doc),
+        contributor: fetch_contributor(doc), relation: fetch_relation(doc),
+        place: fetch_place(doc), series: fetch_series(doc),
+        language: [doc["language"]], script: ["Latn"], doctype: "standard"
+      )
+      write_file item
+    rescue StandardError => e
+      warn "Document: #{doc.at('doi').text}"
+      warn e.message
+      warn e.backtrace[0..5].join("\n")
+      # raise e
+    end
+
+    #
+    # Fetch all the documnts from dataset
+    #
+    def fetch # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      t1 = Time.now
+      puts "Started at: #{t1}"
+
+      docs = Nokogiri::XML OpenURI.open_uri URL
+      FileUtils.mkdir @output unless Dir.exist? @output
+      FileUtils.rm Dir[File.join(@output, "*.#{@ext}")]
+      docs.xpath("/body/query/doi_record/report-paper/report-paper_metadata")
+        .each { |doc| parse_doc doc }
+
+      t2 = Time.now
+      puts "Stopped at: #{t2}"
+      puts "Done in: #{(t2 - t1).round} sec."
+    rescue StandardError => e
+      warn e.message
+      warn e.backtrace[0..5].join("\n")
+    end
+
+    #
+    # Fetch all the documnts from dataset
+    #
+    # @param [String] output foldet name to save the documents
+    # @param [String] format format to save the documents (yaml, xml, bibxml)
+    #
+    def self.fetch(output: "data", format: "yaml")
+      new(output, format).fetch
+    end
+  end
+end

data/lib/relaton_nist/hit_collection.rb

@@ -15,17 +15,19 @@ module RelatonNist
     DATAFILE = File.expand_path "pubs-export.zip", DATAFILEDIR
     GHNISTDATA = "https://raw.githubusercontent.com/relaton/relaton-data-nist/main/data/"
 
-    # @param ref_nbr [String]
-    # @param year [String]
-    # @param opts [Hash]
-    # @option opts [String] :stage
-    def initialize(ref_nbr, year = nil, opts = {}) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
-      super ref_nbr, year
-
-      # /(?<docid>(?:SP|FIPS)\s[0-9-]+)/ =~ text
+    def self.search(text, year = nil, opts = {})
+      new(text, year).search(opts)
+    end
+
+    def search(opts)
       @array = from_json(**opts)
       @array = from_ga unless @array.any?
+      sort_hits!
+    end
 
+    private
+
+    def sort_hits!
       @array.sort! do |a, b|
         if a.sort_value == b.sort_value
           (b.hit[:release_date] - a.hit[:release_date]).to_i
@@ -33,10 +35,9 @@ module RelatonNist
           b.sort_value - a.sort_value
         end
       end
+      self
     end
 
-    private
-
     def from_ga # rubocop:disable Metrics/AbcSize
       fn = text.gsub(%r{[/\s:.]}, "_").upcase
       yaml = OpenURI.open_uri "#{GHNISTDATA}#{fn}.yaml"

data/lib/relaton_nist/nist_bibliography.rb

@@ -14,7 +14,7 @@ module RelatonNist
       # @param text [String]
       # @return [RelatonNist::HitCollection]
       def search(text, year = nil, opts = {})
-        HitCollection.new text, year, opts
+        HitCollection.search text, year, opts
       rescue OpenURI::HTTPError, SocketError, OpenSSL::SSL::SSLError => e
         raise RelatonBib::RequestError, e.message
       end
@@ -28,17 +28,18 @@ module RelatonNist
       # @option opts [TrueClass, FalseClass] :bibdata
       #
       # @return [String] Relaton XML serialisation of reference
-      def get(code, year = nil, opts = {})
-        return fetch_ref_err(code, year, []) if code.match? /\sEP$/
+      def get(code, year = nil, opts = {}) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        return fetch_ref_err(code, year, []) if code.match?(/\sEP$/)
 
-        /^(?<code2>[^\(]+)(\((?<date2>\w+\s(\d{2},\s)?\d{4})\))?\s?\(?((?<=\()(?<stage>[^\)]+))?/ =~ code
+        /^(?<code2>[^(]+)(?:\((?<date2>\w+\s(?:\d{2},\s)?\d{4})\))?\s?\(?(?:(?<=\()(?<stage>[^\)]+))?/ =~ code
         stage ||= /(?<=\.)PD-\w+(?=\.)/.match(code)&.to_s
         if code2
           code = code2.strip
           if date2
-            if /\w+\s\d{4}/.match? date2
+            case date2
+            when /\w+\s\d{4}/
               opts[:issued_date] = Date.strptime date2, "%B %Y"
-            elsif /\w+\s\d{2},\s\d{4}/.match? date2
+            when /\w+\s\d{2},\s\d{4}/
               opts[:updated_date] = Date.strptime date2, "%B %d, %Y"
             end
           end

data/lib/relaton_nist/processor.rb

@@ -2,11 +2,12 @@ require "relaton/processor"
 
 module RelatonNist
   class Processor < Relaton::Processor
-    def initialize
+    def initialize # rubocop:disable Lint/MissingSuper
       @short = :relaton_nist
       @prefix = "NIST"
       @defaultprefix = %r{^(NIST|NISTGCR|ITL Bulletin|JPCRD|NISTIR|CSRC|FIPS)(/[^\s])?\s}
       @idtype = "NIST"
+      @datasets = %w[nist-tech-pubs]
     end
 
     # @param code [String]
@@ -17,6 +18,18 @@ module RelatonNist
       ::RelatonNist::NistBibliography.get(code, date, opts)
     end
 
+    #
+    # Fetch all the documents from a source
+    #
+    # @param [String] _source source name
+    # @param [Hash] opts
+    # @option opts [String] :output directory to output documents
+    # @option opts [String] :format
+    #
+    def fetch_data(_source, opts)
+      DataFetcher.fetch(**opts)
+    end
+
     # @param xml [String]
     # @return [RelatonNist::GbBibliographicItem]
     def from_xml(xml)

data/lib/relaton_nist/scrapper.rb

@@ -62,44 +62,10 @@ module RelatonNist
       # Fetch status.
       # @param doc [Hash]
       # @return [RelatonNist::DocumentStatus]
-      def fetch_status(doc) # , status)
-        # if doc.is_a? Hash
+      def fetch_status(doc)
         stage = doc["status"]
         subst = doc["substage"]
         iter = doc["iteration"] == "initial" ? 1 : doc["iteration"]
-        # else
-        #   case status
-        #   when "draft (obsolete)"
-        #     stage = "draft-public"
-        #     subst = "withdrawn"
-        #   when "retired draft"
-        #     stage = "draft-public"
-        #     subst = "retired"
-        #   when "withdrawn"
-        #     stage = "final"
-        #     subst = "withdrawn"
-        #   when /^draft/
-        #     stage = "draft-public"
-        #     subst = "active"
-        #   else
-        #     stage = status
-        #     subst = "active"
-        #   end
-
-        #   iter = nil
-        #   if stage.include? "draft"
-        #     iter = 1
-        #     history = doc.xpath("//span[@id='pub-history-container']/a"\
-        #                         "|//span[@id='pub-history-container']/span")
-        #     history.each_with_index do |h, idx|
-        #       next if h.name == "a"
-
-        #       iter = idx + 1 if idx.positive?
-        #       break
-        #     end
-        #   end
-        # end
-
         RelatonNist::DocumentStatus.new stage: stage, substage: subst, iteration: iter.to_s
       end
 
@@ -132,7 +98,6 @@ module RelatonNist
         dates
       end
 
-      # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       # @param doc [Hash]
       # @return [Array<RelatonBib::ContributionInfo>]
       def fetch_contributors(doc)
@@ -144,23 +109,12 @@ module RelatonNist
         contribs + contributors_json(
           doc["editors"], "editor", doc["language"], doc["script"]
         )
-        # else
-        #   name = "National Institute of Standards and Technology"
-        #   org = RelatonBib::Organization.new(
-        #     name: name, url: "www.nist.gov", abbreviation: "NIST",
-        #   )
-        #   contribs << RelatonBib::ContributionInfo.new(entity: org, role: [type: "publisher"])
-        #   authors = doc.at('//h4[.="Author(s)"]/following-sibling::p')
-        #   contribs += contributors(authors, "author")
-        #   editors = doc.at('//h4[.="Editor(s)"]/following-sibling::p')
-        #   contribs + contributors(editors, "editor")
-        # end
       end
 
       # @param doc [Array<Hash>]
       # @param role [String]
       # @return [Array<RelatonBib::ContributionInfo>]
-      def contributors_json(doc, role, lang = "en", script = "Latn")
+      def contributors_json(doc, role, lang = "en", script = "Latn") # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
         doc.map do |contr|
           if contr["affiliation"]
             if contr["affiliation"]["acronym"]
@@ -185,43 +139,6 @@ module RelatonNist
         end.compact
       end
 
-      # rubocop:disable Metrics/CyclomaticComplexity
-      # @param doc [Nokogiri::HTML::Element, Array<Hash>]
-      # @param role [String]
-      # @return [Array<RelatonBib::ContributionInfo>]
-      # def contributors(doc, role, lang = "en", script = "Latn")
-      #   return [] if doc.nil?
-
-      #   doc.text.split(", ").map do |contr|
-      #     /(?<an>.+?)(\s+\((?<abbrev>.+?)\))?$/ =~ contr.strip
-      #     if abbrev && an.downcase !~ /(task|force|group)/ && an.split.size.between?(2, 3)
-      #       fullname = RelatonBib::FullName.new(
-      #         completename: RelatonBib::LocalizedString.new(an, lang, script)
-      #       )
-      #       case abbrev
-      #       when "NIST"
-      #         org_name = "National Institute of Standards and Technology"
-      #         url = "www.nist.gov"
-      #       when "MITRE"
-      #         org_name = abbrev
-      #         url = "www.mitre.org"
-      #       else
-      #         org_name = abbrev
-      #         url = nil
-      #       end
-      #       org = RelatonBib::Organization.new name: org_name, url: url, abbreviation: abbrev
-      #       affiliation = RelatonBib::Affiliation.new organization: org
-      #       entity = RelatonBib::Person.new(
-      #         name: fullname, affiliation: [affiliation],
-      #       )
-      #     else
-      #       entity = RelatonBib::Organization.new name: an, abbreviation: abbrev
-      #     end
-      #     RelatonBib::ContributionInfo.new entity: entity, role: [type: role]
-      #   end
-      # end
-      # rubocop:enable Metrics/CyclomaticComplexity, Metrics/AbcSize, Metrics/MethodLength
-
       # @param name [Hash]
       # @param lang [Strong]
       # @param script [String]
@@ -253,80 +170,30 @@ module RelatonNist
         return unless doc["edition"]
 
         rev = doc["edition"]
-        # else
-        #   return unless /(?<=Rev\.\s)(?<rev>\d+)/ =~ doc
-        # end
-
         "Revision #{rev}"
       end
 
-      # Fetch abstracts.
-      # @param doc [Nokigiri::HTML::Document]
-      # @return [Array<Hash>]
-      # def fetch_abstract(doc)
-      #   abstract_content = doc.xpath(
-      #     '//div[contains(@class, "pub-abstract-callout")]/div[1]/p',
-      #   ).text
-      #   [{
-      #     content: abstract_content,
-      #     language: "en",
-      #     script: "Latn",
-      #     format: "text/plain",
-      #   }]
-      # end
-
       # Fetch copyright.
       # @param doc [Nokogiri::HTL::Document, String]
       # @return [Array<Hash>]
       def fetch_copyright(doc)
         name = "National Institute of Standards and Technology"
         url = "www.nist.gov"
-        # d = if doc.is_a? String then doc
-        #     else
-        #       doc.at("//span[@id='pub-release-date']")&.text&.strip
-        #     end
         from = doc&.match(/\d{4}/)&.to_s
         [{ owner: [{ name: name, abbreviation: "NIST", url: url }], from: from }]
       end
 
-      # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
-
       # Fetch links.
       # @param doc [Hash]
       # @return [Array<Hash>]
       def fetch_link(doc)
         links = []
-        # if doc.is_a? Hash
-        links << { type: "uri", content: doc["uri"] } if doc["uri"]
-        doi = "https://doi.org/" + doc["doi"] if doc["doi"]
-        # else
-        #   pub = doc.at "//p/strong[contains(., 'Publication:')]"
-        #   pdf = pub&.at "./following-sibling::a[.=' Local Download']"
-        #   doi = pub&.at("./following-sibling::a[contains(.,'(DOI)')]")&.attr :href
-        #   links << { type: "pdf", content: pdf[:href] } if pdf
-        # end
-        links << { type: "doi", content: doi } if doi
+        links << { type: "src", content: doc["uri"] } if doc["uri"]
+        if doc["doi"]
+          links << { type: "doi", content: "https://doi.org/#{doc['doi']}" }
+        end
         links
       end
-      # rubocop:enable Metrics/MethodLength
-
-      # Fetch relations.
-      # @param doc [Nokogiri::HTML::Document]
-      # @return [Array<RelatonNist::DocumentRelation>]
-      # def fetch_relations(doc)
-      #   relations = doc.xpath('//span[@id="pub-supersedes-container"]/a').map do |r|
-      #     doc_relation "supersedes", r.text, DOMAIN + r[:href]
-      #   end
-
-      #   relations += doc.xpath('//span[@id="pub-part-container"]/a').map do |r|
-      #     doc_relation "partOf", r.text, DOMAIN + r[:href]
-      #   end
-
-      #   relations + doc.xpath('//span[@id="pub-related-container"]/a').map do |r|
-      #     doc_relation "updates", r.text, DOMAIN + r[:href]
-      #   end
-      # end
-      # rubocop:enable Metrics/AbcSize
 
       def fetch_relations_json(doc)
         relations = doc["supersedes"].map do |r|
@@ -354,68 +221,12 @@ module RelatonNist
         )
       end
 
-      # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
-
-      # @param doc [Nokogiri::HTML::Document]
-      # @return [Array<RelatonBib::Series>]
-      # def fetch_series(doc)
-      #   series = doc.xpath "//span[@id='pub-history-container']/a"\
-      #     "|//span[@id='pub-history-container']/span"
-      #   series.map.with_index do |s, idx|
-      #     next if s.name == "span"
-
-      #     iter = if idx.zero? then "I"
-      #            else idx + 1
-      #            end
-
-      #     content = s.text.match(/^[^\(]+/).to_s.strip.squeeze " "
-
-      #     ref = case s.text
-      #           when /^Draft/
-      #             content.match(/(?<=Draft\s).+/).to_s + " (#{iter}PD)"
-      #           when /\(Draft\)/ then content + " (#{iter}PD)"
-      #           else content
-      #           end
-
-      #     fref = RelatonBib::FormattedRef.new(
-      #       content: ref, language: "en", script: "Latn", format: "text/plain",
-      #     )
-      #     RelatonBib::Series.new(formattedref: fref)
-      #   end.select { |s| s }
-      # end
-      # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
-
       # @param doc [Hash]
       # @return [Array<RelatonNist::Keyword>]
       def fetch_keywords(doc)
-        # kws = if doc.is_a? Hash
-        #         doc["keywords"]
-        #       else
-        #         doc.xpath "//span[@id='pub-keywords-container']/span"
-        #       end
         doc["keywords"].map { |kw| kw.is_a?(String) ? kw : kw.text }
       end
 
-      # rubocop:disable Metrics/AbcSize
-      # @param doc [Nokogiri::HTML::Document]
-      # @return [RelatonNist::CommentPeriod, NilClass]
-      # def fetch_commentperiod(doc)
-      #   cp = doc.at "//span[@id='pub-comments-due']"
-      #   return unless cp
-
-      #   to = Date.strptime cp.text.strip, "%B %d, %Y"
-
-      #   d = doc.at("//span[@id='pub-release-date']").text.strip
-      #   from = Date.strptime(d, "%B %Y").to_s
-
-      #   ex = doc.at "//strong[contains(.,'The comment closing date has been "\
-      #   "extended to')]"
-      #   ext = ex&.text&.match(/\w+\s\d{2},\s\d{4}/).to_s
-      #   extended = ext.empty? ? nil : Date.strptime(ext, "%B %d, %Y")
-      #   CommentPeriod.new from: from, to: to, extended: extended
-      # end
-      # rubocop:enable Metrics/AbcSize
-
       # @param json [Hash]
       # @return [RelatonNist::CommentPeriod, NilClass]
       def fetch_commentperiod_json(json)

data/lib/relaton_nist/version.rb

@@ -1,3 +1,3 @@
 module RelatonNist
-  VERSION = "1.9.1".freeze
+  VERSION = "1.9.6".freeze
 end

data/lib/relaton_nist.rb

@@ -1,5 +1,6 @@
 require "relaton_nist/version"
 require "relaton_nist/nist_bibliography"
+require "relaton_nist/data_fetcher"
 
 # if defined? Relaton
 #   require_relative "relaton/processor"

data/relaton_nist.gemspec

@@ -32,6 +32,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "vcr"
   spec.add_development_dependency "webmock"
 
-  spec.add_dependency "relaton-bib", "~> 1.9.0"
+  spec.add_dependency "relaton-bib", ">= 1.9.19"
   spec.add_dependency "rubyzip"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-nist
 version: !ruby/object:Gem::Version
-  version: 1.9.1
+  version: 1.9.6
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-08-30 00:00:00.000000000 Z
+date: 2022-01-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: equivalent-xml
@@ -126,16 +126,16 @@ dependencies:
   name: relaton-bib
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.9.0
+        version: 1.9.19
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.9.0
+        version: 1.9.19
 - !ruby/object:Gem::Dependency
   name: rubyzip
   requirement: !ruby/object:Gem::Requirement
@@ -175,6 +175,7 @@ files:
 - grammars/reqt.rng
 - lib/relaton_nist.rb
 - lib/relaton_nist/comment_period.rb
+- lib/relaton_nist/data_fetcher.rb
 - lib/relaton_nist/document_relation.rb
 - lib/relaton_nist/document_status.rb
 - lib/relaton_nist/hash_converter.rb