relaton-nist 1.9.0 → 1.9.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 51d5205328c024259b5fea0e2632c72eb51045ff119a789e6760a678af3e6961
-  data.tar.gz: dc3da903e2308fa2ebc3cf894a4972e69a4bdd0ce8dfe39ca770295856bab0ad
+  metadata.gz: 847b161ffdf47b18befc71b090e2397e815594f4c20820b057c93608d171487f
+  data.tar.gz: 73748bbb1ff975876b6346f24a89ec235f4224d61aa4f714e13b8b981ece81cd
 SHA512:
-  metadata.gz: 7794caf0f138e6028da3b44345bf6206c4770c21d019205aa84bff604bafe88f1eb888c1126ec99bf6f501494c25eed956080f1c5cc72f7cb46427f2c4933ca8
-  data.tar.gz: d03f8aec87890fc4bac521318306b87d9c92a174389c9a0109b54cfd360cca387d0c806333a7ffa50a8ae2905c41f2dd8ac14c9903666720ad15615f42e9e757
+  metadata.gz: 57640ab1c61feddcf2ac798128d46aff89b190670efa11b624afe9270c94c832c4f53b1cae07dc44f2d189d14b828c1ef1cd1b55d40ac5b94bd65954cd875efa
+  data.tar.gz: 8df687fcfae413e7efa74d6fbe1fbda4835d61b2c6e3751629365497e83e5f288ec5dcebce0b4472193b81a734d5f401babf1472c212370cfc817b8bc7d130bc

data/.gitignore

@@ -7,6 +7,7 @@
 /spec/reports/
 /tmp/
 .vscode/
+/data/
 .rubocop-https---raw-githubusercontent-com-riboseinc-oss-guides-master-ci-rubocop-yml
 
 # rspec failure tracking

data/.rubocop.yml

@@ -2,6 +2,8 @@
 # https://github.com/riboseinc/oss-guides
 # All project-specific additions and overrides should be specified in this file.
 
+require: rubocop-rails
+
 inherit_from:
   - https://raw.githubusercontent.com/riboseinc/oss-guides/master/ci/rubocop.yml
 AllCops:

data/README.adoc

@@ -147,6 +147,17 @@ item.docidentifier.first.id
 => "SP 800-38A-Add"
 ----
 
+=== Typed links
+
+NIST documents may have `src` and `doi` link types.
+
+[source,ruby]
+----
+item.link
+=> [#<RelatonBib::TypedUri:0x00007f901971dc10 @content=#<Addressable::URI:0x62c URI:https://csrc.nist.gov/publications/detail/sp/800-67/rev-2/final>, @type="src">,
+ #<RelatonBib::TypedUri:0x00007f901971d6e8 @content=#<Addressable::URI:0x640 URI:https://doi.org/10.6028/NIST.SP.800-67r2>, @type="doi">]
+----
+
 === Create bibliographic item from YAML
 [source,ruby]
 ----
@@ -159,6 +170,25 @@ RelatonNist::NistBibliographicItem.from_hash hash
 ...
 ----
 
+=== Fetch data
+
+This gem uses the https://raw.githubusercontent.com/usnistgov/NIST-Tech-Pubs/nist-pages/xml/allrecords.xml dataset as one of data sources.
+
+The method `RelatonNist::DataFetcher.fetch(output: "data", format: "yaml")` fetches all the documents from the datast and save them to the `./data` folder in YAML format.
+Arguments:
+
+- `output` - folder to save documents (default './data').
+- `format` - format in which the documents are saved. Possimle formats are: `yaml`, `xml`, `bibxxml` (default `yaml`).
+
+[source,ruby]
+----
+RelatonNist::DataFetcher.fetch
+Started at: 2021-09-01 18:01:01 +0200
+Stopped at: 2021-09-01 18:01:43 +0200
+Done in: 42 sec.
+=> nil
+----
+
 == Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/bin/rspec

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'rspec' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rspec-core", "rspec")

data/lib/relaton_nist/data_fetcher.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module RelatonNist
+  class DataFetcher
+    RELATION_TYPES = {
+      "replaces" => "obsoletes",
+      "isVersionOf" => "editionOf",
+      "hasTranslation" => "hasTranslation",
+      "isTranslationOf" => "translatedFrom",
+      "hasPreprint" => "hasReprint",
+      "isSupplementTo" => "complements",
+    }.freeze
+    URL = "https://raw.githubusercontent.com/usnistgov/NIST-Tech-Pubs/nist-pages/xml/allrecords.xml"
+
+    def initialize(output, format)
+      @output = output
+      @format = format
+      @ext = format.sub(/^bib/, "")
+    end
+
+    def parse_docid(doc)
+      doi = doc.at("doi_data/doi").text
+      id = doc.at("publisher_item/item_number", "publisher_item/identifier").text.sub(%r{^/}, "")
+      case doi
+      when "10.6028/NBS.CIRC.12e2revjune" then id.sub!("13e", "12e")
+      when "10.6028/NBS.CIRC.36e2" then id.sub!("46e", "36e")
+      when "10.6028/NBS.HB.67suppJune1967" then id.sub!("1965", "1967")
+      when "10.6028/NBS.HB.105-1r1990" then id.sub!("105-1-1990", "105-1r1990")
+      when "10.6028/NIST.HB.150-10-1995" then id.sub!(/150-10$/, "150-10-1995")
+      end
+      [{ type: "NIST", id: id }, { type: "DOI", id: doi }]
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<RelatonBib::DocumentIdentifier>]
+    def fetch_docid(doc)
+      parse_docid(doc).map do |id|
+        RelatonBib::DocumentIdentifier.new(type: id[:type], id: id[:id])
+      end
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [RelatonBib::TypedTitleStringCollection, Array]
+    def fetch_title(doc)
+      t = doc.xpath("titles/title|titles/subtitle")
+      return [] unless t.any?
+
+      RelatonBib::TypedTitleString.from_string t.map(&:text).join(" "), "en", "Latn"
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<RelatonBib::BibliographicDate>]
+    def fetch_date(doc)
+      doc.xpath("publication_date|approval_date").map do |dt|
+        on = dt.at("year").text
+        if (m = dt.at "month")
+          on += "-#{m.text}"
+          d = dt.at "day"
+          on += "-#{d.text}" if d
+        end
+        type = dt.name == "publication_date" ? "published" : "confirmed"
+        RelatonBib::BibliographicDate.new(type: type, on: on)
+      end
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [String]
+    def fetch_edition(doc)
+      doc.at("edition_number")&.text
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<Hash>]
+    def fetch_relation(doc)
+      ns = "http://www.crossref.org/relations.xsd"
+      doc.xpath("./ns:program/ns:related_item", ns: ns).map do |rel|
+        doi = rel.at_xpath("ns:intra_work_relation|ns:inter_work_relation", ns: ns)
+        # ref = doi_to_id doi.text
+        # ref, = parse_docid doc
+        fref = RelatonBib::FormattedRef.new content: doi.text
+        bibitem = RelatonBib::BibliographicItem.new formattedref: fref
+        type = RELATION_TYPES[doi["relationship-type"]]
+        { type: type, bibitem: bibitem }
+      end
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<RelatonBib::TypedUri>]
+    def fetch_link(doc)
+      url = doc.at("doi_data/resource").text
+      [RelatonBib::TypedUri.new(type: "doi", content: url)]
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<RelatonBib::FormattedString>]
+    def fetch_abstract(doc)
+      doc.xpath("jats:abstract/jats:p", "jats" => "http://www.ncbi.nlm.nih.gov/JATS1").map do |a|
+        RelatonBib::FormattedString.new(content: a.text, language: doc["language"], script: "Latn")
+      end
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<Hash>]
+    def fetch_contributor(doc) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+      contribs = doc.xpath("contributors/person_name").map do |p|
+        forename = []
+        initial = []
+        p.at("given_name")&.text&.split&.each do |fn|
+          if /^(?<init>\w)\.?$/ =~ fn
+            initial << RelatonBib::LocalizedString.new(init, doc["language"], "Latn")
+          else
+            forename << RelatonBib::LocalizedString.new(fn, doc["language"], "Latn")
+          end
+        end
+        sname = p.at("surname").text
+        surname = RelatonBib::LocalizedString.new sname, doc["language"], "Latn"
+        initial = []
+        ident = p.xpath("ORCID").map do |id|
+          RelatonBib::PersonIdentifier.new "orcid", id.text
+        end
+        fullname = RelatonBib::FullName.new(
+          surname: surname, forename: forename, initial: initial, identifier: ident,
+        )
+        person = RelatonBib::Person.new name: fullname
+        { entity: person, role: [{ type: p["contributor_role"] }] }
+      end
+      contribs + doc.xpath("publisher").map do |p|
+        abbr = p.at("../institution/institution_acronym")&.text
+        org = RelatonBib::Organization.new(name: p.at("publisher_name").text, abbreviation: abbr)
+        { entity: org, role: [{ type: "publisher" }] }
+      end
+    end
+
+    # @param doc [Nokogiri::XML::Element]
+    # @return [Array<String>]
+    def fetch_place(doc)
+      doc.xpath("institution/institution_place").map(&:text)
+    end
+
+    #
+    # Save document
+    #
+    # @param bib [RelatonNist::NistBibliographicItem]
+    #
+    def write_file(bib) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      id = bib.docidentifier[0].id.gsub(%r{[/\s:.]}, "_").upcase.sub(/^NIST_IR/, "NISTIR")
+      file = File.join(@output, "#{id}.#{@ext}")
+      if File.exist? file
+        warn "File #{file} exists. Docid: #{bib.docidentifier[0].id}"
+        # warn "Link: #{bib.link.detect { |l| l.type == 'src' }.content}"
+      else
+        output = case @format
+                 when "yaml" then bib.to_hash.to_yaml
+                 when "xml" then bib.to_xml bibdata: true
+                 else bib.send "to_#{@format}"
+                 end
+        File.write file, output, encoding: "UTF-8"
+      end
+    end
+
+    #
+    # Create a document instance an save it.
+    #
+    # @param doc [Nokogiri::XML::Element]
+    #
+    # @raise [StandardError]
+    #
+    def parse_doc(doc) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
+      # mtd = doc.at('doi_record/report-paper/report-paper_metadata')
+      item = RelatonNist::NistBibliographicItem.new(
+        type: "standard", docid: fetch_docid(doc), title: fetch_title(doc),
+        link: fetch_link(doc), abstract: fetch_abstract(doc),
+        date: fetch_date(doc), edition: fetch_edition(doc),
+        contributor: fetch_contributor(doc), relation: fetch_relation(doc),
+        place: fetch_place(doc),
+        language: [doc["language"]], script: ["Latn"], doctype: "standard"
+      )
+      write_file item
+    rescue StandardError => e
+      warn "Document: #{doc.at('doi').text}"
+      warn e.message
+      raise e
+    end
+
+    #
+    # Fetch all the documnts from dataset
+    #
+    def fetch # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      t1 = Time.now
+      puts "Started at: #{t1}"
+
+      docs = Nokogiri::XML OpenURI.open_uri URL
+      FileUtils.mkdir @output unless Dir.exist? @output
+      FileUtils.rm Dir[File.join(@output, "*.#{@ext}")]
+      docs.xpath("/body/query/doi_record/report-paper/report-paper_metadata")
+        .each { |doc| parse_doc doc }
+
+      t2 = Time.now
+      puts "Stopped at: #{t2}"
+      puts "Done in: #{(t2 - t1).round} sec."
+    rescue StandardError => e
+      warn e.message
+    end
+
+    #
+    # Fetch all the documnts from dataset
+    #
+    # @param [String] output foldet name to save the documents
+    # @param [String] format format to save the documents (yaml, xml, bibxml)
+    #
+    def self.fetch(output: "data", format: "yaml")
+      new(output, format).fetch
+    end
+  end
+end

data/lib/relaton_nist/hit.rb

@@ -3,6 +3,8 @@
 module RelatonNist
   # Hit.
   class Hit < RelatonBib::Hit
+    attr_writer :fetch
+
     # Parse page.
     # @return [RelatonNist::NistBliographicItem]
     def fetch
@@ -10,7 +12,7 @@ module RelatonNist
     end
 
     # @return [Iteger]
-    def sort_value
+    def sort_value # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
       @sort_value ||= begin
         sort_phrase = [hit[:serie], hit[:code], hit[:title]].join " "
         corr = hit_collection&.text&.split&.map do |w|

data/lib/relaton_nist/hit_collection.rb

@@ -13,18 +13,21 @@ module RelatonNist
     PUBS_EXPORT = URI.join(DOMAIN, "/CSRC/media/feeds/metanorma/pubs-export")
     DATAFILEDIR = File.expand_path ".relaton/nist", Dir.home
     DATAFILE = File.expand_path "pubs-export.zip", DATAFILEDIR
+    GHNISTDATA = "https://raw.githubusercontent.com/relaton/relaton-data-nist/main/data/"
 
-    # @param ref_nbr [String]
-    # @param year [String]
-    # @param opts [Hash]
-    # @option opts [String] :stage
-    def initialize(ref_nbr, year = nil, opts = {}) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
-      super ref_nbr, year
+    def self.search(text, year = nil, opts = {})
+      new(text, year).search(opts)
+    end
 
-      /(?<docid>(?:SP|FIPS)\s[0-9-]+)/ =~ text
-      @array = docid ? from_json(docid, **opts) : from_csrc(**opts)
-      @array = from_csrc(**opts) unless @array.any?
+    def search(opts)
+      @array = from_json(**opts)
+      @array = from_ga unless @array.any?
+      sort_hits!
+    end
 
+    private
+
+    def sort_hits!
       @array.sort! do |a, b|
         if a.sort_value == b.sort_value
           (b.hit[:release_date] - a.hit[:release_date]).to_i
@@ -32,56 +35,68 @@ module RelatonNist
           b.sort_value - a.sort_value
         end
       end
+      self
     end
 
-    private
+    def from_ga # rubocop:disable Metrics/AbcSize
+      fn = text.gsub(%r{[/\s:.]}, "_").upcase
+      yaml = OpenURI.open_uri "#{GHNISTDATA}#{fn}.yaml"
+      hash = YAML.safe_load yaml
+      bib = RelatonNist::NistBibliographicItem.from_hash hash
+      hit = Hit.new({ code: text }, self)
+      hit.fetch = bib
+      [hit]
+    rescue OpenURI::HTTPError => e
+      return [] if e.io.status[0] == "404"
+
+      raise e
+    end
 
     # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
 
     # @param stage [String]
     # @return [Array<RelatonNist::Hit>]
-    def from_csrc(**opts)
-      from, to = nil
-      if year
-        d    = Date.strptime year, "%Y"
-        from = d.strftime "%m/%d/%Y"
-        to   = d.next_year.prev_day.strftime "%m/%d/%Y"
-      end
-      url = "#{DOMAIN}/publications/search?keywords-lg=#{text}"\
-        "&sortBy-lg=relevence"
-      url += "&dateFrom-lg=#{from}" if from
-      url += "&dateTo-lg=#{to}" if to
-      url += if /PD/.match? opts[:stage]
-               "&status-lg=Draft,Retired Draft,Withdrawn"
-             else
-               "&status-lg=Final,Withdrawn"
-             end
-
-      doc = Nokogiri::HTML OpenURI.open_uri(::Addressable::URI.parse(url).normalize)
-      doc.css("table.publications-table > tbody > tr").map do |h|
-        link  = h.at("td/div/strong/a")
-        serie = h.at("td[1]").text.strip
-        code  = h.at("td[2]").text.strip
-        title = link.text
-        doc_url = DOMAIN + link[:href]
-        status = h.at("td[4]").text.strip.downcase
-        release_date = Date.strptime h.at("td[5]").text.strip, "%m/%d/%Y"
-        Hit.new(
-          {
-            code: code, serie: serie, title: title, url: doc_url,
-            status: status, release_date: release_date
-          }, self
-        )
-      end
-    end
+    # def from_csrc(**opts)
+    #   from, to = nil
+    #   if year
+    #     d    = Date.strptime year, "%Y"
+    #     from = d.strftime "%m/%d/%Y"
+    #     to   = d.next_year.prev_day.strftime "%m/%d/%Y"
+    #   end
+    #   url = "#{DOMAIN}/publications/search?keywords-lg=#{text}"\
+    #         "&sortBy-lg=relevence"
+    #   url += "&dateFrom-lg=#{from}" if from
+    #   url += "&dateTo-lg=#{to}" if to
+    #   url += if /PD/.match? opts[:stage]
+    #            "&status-lg=Draft,Retired Draft,Withdrawn"
+    #          else
+    #            "&status-lg=Final,Withdrawn"
+    #          end
+
+    #   doc = Nokogiri::HTML OpenURI.open_uri(::Addressable::URI.parse(url).normalize)
+    #   doc.css("table.publications-table > tbody > tr").map do |h|
+    #     link  = h.at("td/div/strong/a")
+    #     serie = h.at("td[1]").text.strip
+    #     code  = h.at("td[2]").text.strip
+    #     title = link.text
+    #     doc_url = DOMAIN + link[:href]
+    #     status = h.at("td[4]").text.strip.downcase
+    #     release_date = Date.strptime h.at("td[5]").text.strip, "%m/%d/%Y"
+    #     Hit.new(
+    #       {
+    #         code: code, serie: serie, title: title, url: doc_url,
+    #         status: status, release_date: release_date
+    #       }, self
+    #     )
+    #   end
+    # end
     # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
     # Fetches data form json
-    # @param docid [String]
     # @param stage [String]
     # @return [Array<RelatonNist::Hit>]
-    def from_json(docid, **opts)
-      select_data(docid, **opts).map do |h|
+    def from_json(**opts)
+      select_data(**opts).map do |h|
         /(?<serie>(?<=-)\w+$)/ =~ h["series"]
         title = [h["title-main"], h["title-sub"]].compact.join " - "
         release_date = RelatonBib.parse_date h["published-date"], false
@@ -91,10 +106,9 @@ module RelatonNist
       end
     end
 
-    # @param docid [String]
     # @param stage [String]
     # @return [Array<Hach>]
-    def select_data(docid, **opts) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength,Metrics/PerceivedComplexity
+    def select_data(**opts) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength,Metrics/PerceivedComplexity
       d = Date.strptime year, "%Y" if year
       statuses = %w[draft-public draft-prelim]
       data.select do |doc|
@@ -105,7 +119,7 @@ module RelatonNist
         else
           next unless doc["status"] == "final"
         end
-        doc["docidentifier"].include? docid
+        doc["docidentifier"].include? text
       end
     end
 

data/lib/relaton_nist/nist_bibliographic_item.rb

@@ -67,7 +67,7 @@ module RelatonNist
     # @return [RelatonNist::GbBibliographicItem]
     def self.from_hash(hash)
       item_hash = RelatonNist::HashConverter.hash_to_bib(hash)
-      new **item_hash
+      new(**item_hash)
     end
 
     # @param opts [Hash]

data/lib/relaton_nist/nist_bibliography.rb

@@ -14,9 +14,9 @@ module RelatonNist
       # @param text [String]
       # @return [RelatonNist::HitCollection]
       def search(text, year = nil, opts = {})
-        HitCollection.new text, year, opts
-      rescue OpenURI::HTTPError, SocketError, OpenSSL::SSL::SSLError
-        raise RelatonBib::RequestError, "Could not access https://www.nist.gov"
+        HitCollection.search text, year, opts
+      rescue OpenURI::HTTPError, SocketError, OpenSSL::SSL::SSLError => e
+        raise RelatonBib::RequestError, e.message
       end
 
       # @param code [String] the NIST standard Code to look up (e..g "8200")
@@ -174,7 +174,17 @@ module RelatonNist
         }
         ref = matches[:code] ? "#{matches[:serie]} #{matches[:code]}" : code
         result = search(ref, year, opts)
-        result.select { |i| search_filter i, matches, code }
+        selected_result = result.select { |i| search_filter i, matches, code }
+        return selected_result if selected_result.any? || !matches[:code]
+
+        search full_ref(matches)
+      end
+
+      def full_ref(matches)
+        ref = "#{matches[:serie]} #{matches[:code]}"
+        ref += long_to_short(matches[:prt1], matches[:prt2]).to_s
+        ref += long_to_short(matches[:vol1], matches[:vol2]).to_s
+        ref
       end
 
       def match(regex, code)
@@ -192,11 +202,11 @@ module RelatonNist
           (?<code>[0-9-]{3,}[A-Z]?)
           (?<prt1>pt\d+)?
           (?<vol1>v\d+)?
-          (?<ver1>ver[\d\.]+)?
+          (?<ver1>ver[\d.]+)?
           (?<rev1>r\d+)?
           (\s(?<prt2>Part\s\d+))?
           (\s(?<vol2>Vol\.\s\d+))?
-          (\s(?<ver2>(Ver\.|Version)\s[\d\.]+))?
+          (\s(?<ver2>(Ver\.|Version)\s[\d.]+))?
           (\s(?<rev2>Rev\.\s\d+))?
           (\s(?<add>Add)endum)?
         }x =~ item.hit[:code]

data/lib/relaton_nist/processor.rb

@@ -2,11 +2,12 @@ require "relaton/processor"
 
 module RelatonNist
   class Processor < Relaton::Processor
-    def initialize
+    def initialize # rubocop:disable Lint/MissingSuper
       @short = :relaton_nist
       @prefix = "NIST"
       @defaultprefix = %r{^(NIST|NISTGCR|ITL Bulletin|JPCRD|NISTIR|CSRC|FIPS)(/[^\s])?\s}
       @idtype = "NIST"
+      @datasets = %w[nist-tech-pubs]
     end
 
     # @param code [String]
@@ -17,6 +18,18 @@ module RelatonNist
       ::RelatonNist::NistBibliography.get(code, date, opts)
     end
 
+    #
+    # Fetch all the documents from a source
+    #
+    # @param [String] _source source name
+    # @param [Hash] opts
+    # @option opts [String] :output directory to output documents
+    # @option opts [String] :format
+    #
+    def fetch_data(_source, opts)
+      DataFetcher.fetch(**opts)
+    end
+
     # @param xml [String]
     # @return [RelatonNist::GbBibliographicItem]
     def from_xml(xml)

data/lib/relaton_nist/scrapper.rb

@@ -11,23 +11,17 @@ module RelatonNist
       # @param hit_data [Hash]
       # @return [Hash]
       def parse_page(hit_data)
-        item_data = if hit_data[:json]
-                      from_json hit_data
-                    else
-                      from_csrs hit_data
-                    end
-        # doctype = "standard"
+        item_data = from_json hit_data
         titles = fetch_titles(hit_data)
         unless /^(SP|NISTIR|FIPS) /.match? item_data[:docid][0].id
-          # doctype = id_cleanup(item_data[:docid][0].id)
           item_data[:docid][0] = RelatonBib::DocumentIdentifier.new(
-            id: titles[0][:content].upcase, type: "NIST"
+            id: titles[0][:content].upcase, type: "NIST",
           )
         end
         item_data[:fetched] = Date.today.to_s
         item_data[:type] = "standard"
         item_data[:title] = titles
-        item_data[:doctype] = "standard" # doctype
+        item_data[:doctype] = "standard"
 
         NistBibliographicItem.new(**item_data)
       end
@@ -44,7 +38,7 @@ module RelatonNist
           edition: fetch_edition(json),
           language: [json["language"]],
           script: [json["script"]],
-          docstatus: fetch_status(json, hit_data[:status]),
+          docstatus: fetch_status(json), # hit_data[:status]),
           copyright: fetch_copyright(json["published-date"]),
           relation: fetch_relations_json(json),
           place: ["Gaithersburg, MD"],
@@ -53,120 +47,25 @@ module RelatonNist
         }
       end
 
-      def from_csrs(hit_data)
-        doc = get_page hit_data[:url]
-        {
-          # id: fetch_id(doc),
-          link: fetch_link(doc),
-          docid: fetch_docid(doc),
-          date: fetch_dates(doc, hit_data[:release_date]),
-          contributor: fetch_contributors(doc),
-          edition: fetch_edition(hit_data[:code]),
-          language: ["en"],
-          script: ["Latn"],
-          abstract: fetch_abstract(doc),
-          docstatus: fetch_status(doc, hit_data[:status]),
-          copyright: fetch_copyright(doc),
-          relation: fetch_relations(doc),
-          series: fetch_series(doc),
-          keyword: fetch_keywords(doc),
-          commentperiod: fetch_commentperiod(doc),
-        }
-      end
       # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
-      # Strip status from doc id
-      # @param id String
-      # @return String
-      # def id_cleanup(id)
-      #   id.sub(/ \(WITHDRAWN\)/, "").sub(/ \(([^) ]+ )?DRAFT\)/i, "")
-      # end
-
-      # Get page.
-      # @param path [String] page's path
-      # @return [Array<Nokogiri::HTML::Document, String>]
-      def get_page(url)
-        uri = URI url
-        resp = Net::HTTP.get_response(uri)
-        %r{(?<=newLocation = 'https://' \+ window.location.hostname \+ ')(?<path>[^']+)} =~ resp.body
-        if path
-          uri = URI HitCollection::DOMAIN + path
-          resp = Net::HTTP.get_response(uri)
-        end
-        Nokogiri::HTML(resp.body)
-      rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET,
-             EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
-             Net::ProtocolError, OpenSSL::SSL::SSLError
-        raise RelatonBib::RequestError, "Could not access #{url}"
-      end
-
       # Fetch docid.
-      # @param doc [Nokogiri::HTML::Document, String]
+      # @param docid [String]
       # @return [Array<RelatonBib::DocumentIdentifier>]
-      def fetch_docid(doc)
-        item_ref = if doc.is_a? String then doc
-                   else
-                     doc.at(
-                       "//div[contains(@class, 'publications-detail')]/h3"
-                     )&.text&.strip&.sub(/(?<=\w)\([^\)]+\)$/) do |m|
-                       " " + m.upcase
-                     end&.squeeze(" ")&.gsub(/&#13;|\n|\r/, "")
-                   end
-        item_ref ||= "?"
-        item_ref.sub! /\sAddendum$/, "-Add"
+      def fetch_docid(docid)
+        item_ref = docid
+        # item_ref ||= "?"
+        item_ref.sub!(/\sAddendum$/, "-Add")
         [RelatonBib::DocumentIdentifier.new(id: item_ref, type: "NIST")]
       end
 
-      # Fetch id.
-      # @param doc [Nokogiri::HTML::Document]
-      # @return [String]
-      # def fetch_id(doc)
-      #   doc.at("//div[contains(@class, 'publications-detail')]/h3").text.
-      #     strip.gsub(/\s/, "")
-      # end
-
       # Fetch status.
-      # @param doc [Nokogiri::HTML::Document, Hash]
-      # @param status [String]
+      # @param doc [Hash]
       # @return [RelatonNist::DocumentStatus]
-      def fetch_status(doc, status)
-        if doc.is_a? Hash
-          stage = doc["status"]
-          subst = doc["substage"]
-          iter = doc["iteration"] == "initial" ? 1 : doc["iteration"]
-        else
-          case status
-          when "draft (obsolete)"
-            stage = "draft-public"
-            subst = "withdrawn"
-          when "retired draft"
-            stage = "draft-public"
-            subst = "retired"
-          when "withdrawn"
-            stage = "final"
-            subst = "withdrawn"
-          when /^draft/
-            stage = "draft-public"
-            subst = "active"
-          else
-            stage = status
-            subst = "active"
-          end
-
-          iter = nil
-          if stage.include? "draft"
-            iter = 1
-            history = doc.xpath("//span[@id='pub-history-container']/a"\
-                                "|//span[@id='pub-history-container']/span")
-            history.each_with_index do |h, idx|
-              next if h.name == "a"
-
-              iter = idx + 1 if idx.positive?
-              break
-            end
-          end
-        end
-
+      def fetch_status(doc)
+        stage = doc["status"]
+        subst = doc["substage"]
+        iter = doc["iteration"] == "initial" ? 1 : doc["iteration"]
         RelatonNist::DocumentStatus.new stage: stage, substage: subst, iteration: iter.to_s
       end
 
@@ -179,55 +78,43 @@ module RelatonNist
       end
 
       # Fetch dates
-      # @param doc [Nokogiri::HTML::Document]
+      # @param doc [Hash]
       # @param release_date [Date]
       # @return [Array<Hash>]
       def fetch_dates(doc, release_date) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
         dates = [{ type: "published", on: release_date.to_s }]
 
-        if doc.is_a? Hash
-          issued = RelatonBib.parse_date doc["issued-date"]
-          updated = RelatonBib.parse_date doc["updated-date"]
-          dates << { type: "updated", on: updated.to_s } if updated
-          obsoleted = RelatonBib.parse_date doc["obsoleted-date"]
-          dates << { type: "obsoleted", on: obsoleted.to_s } if obsoleted
-        else
-          d = doc.at("//span[@id='pub-release-date']")&.text&.strip
-          issued = RelatonBib.parse_date d
-        end
+        # if doc.is_a? Hash
+        issued = RelatonBib.parse_date doc["issued-date"]
+        updated = RelatonBib.parse_date doc["updated-date"]
+        dates << { type: "updated", on: updated.to_s } if updated
+        obsoleted = RelatonBib.parse_date doc["obsoleted-date"]
+        dates << { type: "obsoleted", on: obsoleted.to_s } if obsoleted
+        # else
+        #   d = doc.at("//span[@id='pub-release-date']")&.text&.strip
+        #   issued = RelatonBib.parse_date d
+        # end
         dates << { type: "issued", on: issued.to_s }
         dates
       end
 
-      # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-      # @param doc [Nokogiri::HTML::Document, Hash]
+      # @param doc [Hash]
       # @return [Array<RelatonBib::ContributionInfo>]
       def fetch_contributors(doc)
         contribs = []
-        if doc.is_a? Hash
-          contribs += contributors_json(
-            doc["authors"], "author", doc["language"], doc["script"]
-          )
-          contribs + contributors_json(
-            doc["editors"], "editor", doc["language"], doc["script"]
-          )
-        else
-          name = "National Institute of Standards and Technology"
-          org = RelatonBib::Organization.new(
-            name: name, url: "www.nist.gov", abbreviation: "NIST",
-          )
-          contribs << RelatonBib::ContributionInfo.new(entity: org, role: [type: "publisher"])
-          authors = doc.at('//h4[.="Author(s)"]/following-sibling::p')
-          contribs += contributors(authors, "author")
-          editors = doc.at('//h4[.="Editor(s)"]/following-sibling::p')
-          contribs + contributors(editors, "editor")
-        end
+        # if doc.is_a? Hash
+        contribs += contributors_json(
+          doc["authors"], "author", doc["language"], doc["script"]
+        )
+        contribs + contributors_json(
+          doc["editors"], "editor", doc["language"], doc["script"]
+        )
       end
 
       # @param doc [Array<Hash>]
       # @param role [String]
       # @return [Array<RelatonBib::ContributionInfo>]
-      def contributors_json(doc, role, lang = "en", script = "Latn")
+      def contributors_json(doc, role, lang = "en", script = "Latn") # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
         doc.map do |contr|
           if contr["affiliation"]
             if contr["affiliation"]["acronym"]
@@ -252,43 +139,6 @@ module RelatonNist
         end.compact
       end
 
-      # rubocop:disable Metrics/CyclomaticComplexity
-      # @param doc [Nokogiri::HTML::Element, Array<Hash>]
-      # @param role [String]
-      # @return [Array<RelatonBib::ContributionInfo>]
-      def contributors(doc, role, lang = "en", script = "Latn")
-        return [] if doc.nil?
-
-        doc.text.split(", ").map do |contr|
-          /(?<an>.+?)(\s+\((?<abbrev>.+?)\))?$/ =~ contr.strip
-          if abbrev && an.downcase !~ /(task|force|group)/ && an.split.size.between?(2, 3)
-            fullname = RelatonBib::FullName.new(
-              completename: RelatonBib::LocalizedString.new(an, lang, script)
-            )
-            case abbrev
-            when "NIST"
-              org_name = "National Institute of Standards and Technology"
-              url = "www.nist.gov"
-            when "MITRE"
-              org_name = abbrev
-              url = "www.mitre.org"
-            else
-              org_name = abbrev
-              url = nil
-            end
-            org = RelatonBib::Organization.new name: org_name, url: url, abbreviation: abbrev
-            affiliation = RelatonBib::Affiliation.new organization: org
-            entity = RelatonBib::Person.new(
-              name: fullname, affiliation: [affiliation],
-            )
-          else
-            entity = RelatonBib::Organization.new name: an, abbreviation: abbrev
-          end
-          RelatonBib::ContributionInfo.new entity: entity, role: [type: role]
-        end
-      end
-      # rubocop:enable Metrics/CyclomaticComplexity, Metrics/AbcSize, Metrics/MethodLength
-
       # @param name [Hash]
       # @param lang [Strong]
       # @param script [String]
@@ -313,87 +163,37 @@ module RelatonNist
         [RelatonBib::LocalizedString.new(part, lang, script)]
       end
 
-      # @param doc [String, Hash]
+      # @param doc [Hash]
       # @return [String, NilClass]
       def fetch_edition(doc)
-        if doc.is_a? Hash
-          return unless doc["edition"]
-
-          rev = doc["edition"]
-        else
-          return unless /(?<=Rev\.\s)(?<rev>\d+)/ =~ doc
-        end
+        # if doc.is_a? Hash
+        return unless doc["edition"]
 
+        rev = doc["edition"]
         "Revision #{rev}"
       end
 
-      # Fetch abstracts.
-      # @param doc [Nokigiri::HTML::Document]
-      # @return [Array<Hash>]
-      def fetch_abstract(doc)
-        abstract_content = doc.xpath(
-          '//div[contains(@class, "pub-abstract-callout")]/div[1]/p',
-        ).text
-        [{
-          content: abstract_content,
-          language: "en",
-          script: "Latn",
-          format: "text/plain",
-        }]
-      end
-
       # Fetch copyright.
       # @param doc [Nokogiri::HTL::Document, String]
       # @return [Array<Hash>]
       def fetch_copyright(doc)
         name = "National Institute of Standards and Technology"
         url = "www.nist.gov"
-        d = if doc.is_a? String then doc
-            else
-              doc.at("//span[@id='pub-release-date']")&.text&.strip
-            end
-        from = d&.match(/\d{4}/)&.to_s
+        from = doc&.match(/\d{4}/)&.to_s
         [{ owner: [{ name: name, abbreviation: "NIST", url: url }], from: from }]
       end
 
-      # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
-
       # Fetch links.
-      # @param doc [Nokogiri::HTML::Document, Hash]
+      # @param doc [Hash]
       # @return [Array<Hash>]
       def fetch_link(doc)
         links = []
-        if doc.is_a? Hash
-          links << { type: "uri", content: doc["uri"] } if doc["uri"]
-          doi = "https://doi.org/" + doc["doi"] if doc["doi"]
-        else
-          pub = doc.at "//p/strong[contains(., 'Publication:')]"
-          pdf = pub&.at "./following-sibling::a[.=' Local Download']"
-          doi = pub&.at("./following-sibling::a[contains(.,'(DOI)')]")&.attr :href
-          links << { type: "pdf", content: pdf[:href] } if pdf
+        links << { type: "src", content: doc["uri"] } if doc["uri"]
+        if doc["doi"]
+          links << { type: "doi", content: "https://doi.org/#{doc['doi']}" }
         end
-        links << { type: "doi", content: doi } if doi
         links
       end
-      # rubocop:enable Metrics/MethodLength
-
-      # Fetch relations.
-      # @param doc [Nokogiri::HTML::Document]
-      # @return [Array<RelatonNist::DocumentRelation>]
-      def fetch_relations(doc)
-        relations = doc.xpath('//span[@id="pub-supersedes-container"]/a').map do |r|
-          doc_relation "supersedes", r.text, DOMAIN + r[:href]
-        end
-
-        relations += doc.xpath('//span[@id="pub-part-container"]/a').map do |r|
-          doc_relation "partOf", r.text, DOMAIN + r[:href]
-        end
-
-        relations + doc.xpath('//span[@id="pub-related-container"]/a').map do |r|
-          doc_relation "updates", r.text, DOMAIN + r[:href]
-        end
-      end
-      # rubocop:enable Metrics/AbcSize
 
       def fetch_relations_json(doc)
         relations = doc["supersedes"].map do |r|
@@ -421,67 +221,11 @@ module RelatonNist
         )
       end
 
-      # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
-
-      # @param doc [Nokogiri::HTML::Document]
-      # @return [Array<RelatonBib::Series>]
-      def fetch_series(doc)
-        series = doc.xpath "//span[@id='pub-history-container']/a"\
-          "|//span[@id='pub-history-container']/span"
-        series.map.with_index do |s, idx|
-          next if s.name == "span"
-
-          iter = if idx.zero? then "I"
-                 else idx + 1
-                 end
-
-          content = s.text.match(/^[^\(]+/).to_s.strip.squeeze " "
-
-          ref = case s.text
-                when /^Draft/
-                  content.match(/(?<=Draft\s).+/).to_s + " (#{iter}PD)"
-                when /\(Draft\)/ then content + " (#{iter}PD)"
-                else content
-                end
-
-          fref = RelatonBib::FormattedRef.new(
-            content: ref, language: "en", script: "Latn", format: "text/plain",
-          )
-          RelatonBib::Series.new(formattedref: fref)
-        end.select { |s| s }
-      end
-      # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
-
-      # @param doc [Nokogiri::HTML::Document, Hash]
+      # @param doc [Hash]
       # @return [Array<RelatonNist::Keyword>]
       def fetch_keywords(doc)
-        kws = if doc.is_a? Hash
-                doc["keywords"]
-              else
-                doc.xpath "//span[@id='pub-keywords-container']/span"
-              end
-        kws.map { |kw| kw.is_a?(String) ? kw : kw.text }
-      end
-
-      # rubocop:disable Metrics/AbcSize
-      # @param doc [Nokogiri::HTML::Document]
-      # @return [RelatonNist::CommentPeriod, NilClass]
-      def fetch_commentperiod(doc)
-        cp = doc.at "//span[@id='pub-comments-due']"
-        return unless cp
-
-        to = Date.strptime cp.text.strip, "%B %d, %Y"
-
-        d = doc.at("//span[@id='pub-release-date']").text.strip
-        from = Date.strptime(d, "%B %Y").to_s
-
-        ex = doc.at "//strong[contains(.,'The comment closing date has been "\
-        "extended to')]"
-        ext = ex&.text&.match(/\w+\s\d{2},\s\d{4}/).to_s
-        extended = ext.empty? ? nil : Date.strptime(ext, "%B %d, %Y")
-        CommentPeriod.new from: from, to: to, extended: extended
+        doc["keywords"].map { |kw| kw.is_a?(String) ? kw : kw.text }
       end
-      # rubocop:enable Metrics/AbcSize
 
       # @param json [Hash]
       # @return [RelatonNist::CommentPeriod, NilClass]

data/lib/relaton_nist/version.rb

@@ -1,3 +1,3 @@
 module RelatonNist
-  VERSION = "1.9.0".freeze
+  VERSION = "1.9.4".freeze
 end

data/lib/relaton_nist.rb

@@ -1,5 +1,6 @@
 require "relaton_nist/version"
 require "relaton_nist/nist_bibliography"
+require "relaton_nist/data_fetcher"
 
 # if defined? Relaton
 #   require_relative "relaton/processor"

data/relaton_nist.gemspec

@@ -25,13 +25,13 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "equivalent-xml", "~> 0.6"
   spec.add_development_dependency "pry-byebug"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "ruby-jing"
   spec.add_development_dependency "simplecov"
   spec.add_development_dependency "vcr"
   spec.add_development_dependency "webmock"
 
-  spec.add_dependency "relaton-bib", "~> 1.9.0"
+  spec.add_dependency "relaton-bib", ">= 1.9.19"
   spec.add_dependency "rubyzip"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-nist
 version: !ruby/object:Gem::Version
-  version: 1.9.0
+  version: 1.9.4
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-08-26 00:00:00.000000000 Z
+date: 2022-01-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: equivalent-xml
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -126,16 +126,16 @@ dependencies:
   name: relaton-bib
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.9.0
+        version: 1.9.19
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.9.0
+        version: 1.9.19
 - !ruby/object:Gem::Dependency
   name: rubyzip
   requirement: !ruby/object:Gem::Requirement
@@ -166,6 +166,7 @@ files:
 - README.adoc
 - Rakefile
 - bin/console
+- bin/rspec
 - bin/setup
 - grammars/basicdoc.rng
 - grammars/biblio.rng
@@ -174,6 +175,7 @@ files:
 - grammars/reqt.rng
 - lib/relaton_nist.rb
 - lib/relaton_nist/comment_period.rb
+- lib/relaton_nist/data_fetcher.rb
 - lib/relaton_nist/document_relation.rb
 - lib/relaton_nist/document_status.rb
 - lib/relaton_nist/hash_converter.rb