relaton-nist 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: 2ab3d1123704cbe701f2f8158ee2d60be05a762141a8c10f2d9f196f8b905dab
-  data.tar.gz: f06320c0918cf4d5b6354015e370b2cc7bab93247bfd842875086005de9ca247
+SHA1:
+  metadata.gz: 42ae6073ff5e1cbaba11be06d27c9e0da310bcf4
+  data.tar.gz: fd4e493fc0f7f3edefe2af58f26a76954c2b1e2c
 SHA512:
-  metadata.gz: 6a1bb823322716ac5115428bedbcad2cee3667bc24b8f7fc02030856f5028b92377196eaa8b4c1c6afe6cd67357af4a9d34d4fe7d09b8313e3310f6cc02394b8
-  data.tar.gz: 4fddf90e843ed03dc7d2dc281674a5ecf0ce69d58509682a36f39044dcfb7a769378153f194872f2199c1c8b702a6ca3c88be6f618d22a28283814ead733c37c
+  metadata.gz: 51ca4ccf407bb4355f669fc82ea2950a720f75af506174a58b807c86e109b21808fc4f380363258941c37328e544f9658c75d9ee24966124f3532f0abf8df421
+  data.tar.gz: e23643ecd8e7a685f2542660bf064ef30ff027460b9898aa6a1435f21f7a2ec4b311bdaf7306ed078e920b5b728d7b46402d368d9d6535bc968474793e30ad1a

data/Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    relaton-nist (0.2.1)
+    relaton-nist (0.2.2)
       relaton-bib (~> 0.2.0)
+      rubyzip
 
 GEM
   remote: https://rubygems.org/
@@ -32,9 +33,9 @@ GEM
     pry-byebug (3.7.0)
       byebug (~> 11.0)
       pry (~> 0.10)
-    public_suffix (3.1.0)
+    public_suffix (3.1.1)
     rake (10.5.0)
-    relaton-bib (0.2.1)
+    relaton-bib (0.2.3)
       addressable
       nokogiri (~> 1.8.4)
     rspec (3.8.0)
@@ -52,6 +53,7 @@ GEM
     rspec-support (3.8.2)
     ruby-debug-ide (0.7.0)
       rake (>= 0.8.1)
+    rubyzip (1.2.3)
     safe_yaml (1.0.5)
     simplecov (0.16.1)
       docile (~> 1.1)

data/lib/relaton_nist.rb

@@ -8,5 +8,20 @@ end
 
 module RelatonNist
   class Error < StandardError; end
-  # Your code goes here...
+
+  class << self
+    # @param date [String]
+    # @return [Date, NilClass]
+    def parse_date(sdate)
+      if /(?<date>\w+\s\d{4})/ =~ sdate # February 2012
+        Date.strptime(date, "%B %Y")
+      elsif /(?<date>\w+\s\d{1,2},\s\d{4})/ =~ sdate # February 11, 2012
+        Date.strptime(date, "%B %d, %Y")
+      elsif /(?<date>\d{4}-\d{2}-\d{2})/ =~ sdate # 2012-02-11
+        Date.parse(date)
+      elsif /(?<date>\d{4}-\d{2})/ =~ sdate # 2012-02
+        Date.strptime date, "%Y-%m"
+      end
+    end
+  end
 end

data/lib/relaton_nist/hit_collection.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "zip"
+require "fileutils"
 require "relaton_nist/hit"
 require "addressable/uri"
 require "open-uri"
@@ -7,8 +9,8 @@ require "open-uri"
 module RelatonNist
   # Page of hit collection.
   class HitCollection < Array
-
     DOMAIN = "https://csrc.nist.gov"
+    DATAFILE = File.expand_path "data/pubs-export.zip", __dir__
 
     # @return [TrueClass, FalseClass]
     attr_reader :fetched
@@ -28,13 +30,58 @@ module RelatonNist
     def initialize(ref_nbr, year = nil, opts = {})
       @text = ref_nbr
       @year = year
+
+      /(?<docid>(SP|FIPS)\s[0-9-]+)/ =~ text
+      hits = docid ? from_json(docid, **opts) : from_csrc(**opts)
+
+      hits.sort! do |a, b|
+        if a.sort_value != b.sort_value
+          b.sort_value - a.sort_value
+        else
+          (b.hit[:release_date] - a.hit[:release_date]).to_i
+        end
+      end
+      concat hits
+      @fetched = false
+    end
+    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+
+    # @return [Iecbib::HitCollection]
+    def fetch
+      workers = RelatonBib::WorkersPool.new 4
+      workers.worker(&:fetch)
+      each do |hit|
+        workers << hit
+      end
+      workers.end
+      workers.result
+      @fetched = true
+      self
+    end
+
+    def to_s
+      inspect
+    end
+
+    # @return [String]
+    def inspect
+      "<#{self.class}:#{format('%#.14x', object_id << 1)} @fetched=#{@fetched}>"
+    end
+
+    private
+
+    # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+
+    # @param stage [String]
+    # @return [Array<RelatonNist::Hit>]
+    def from_csrc(**opts)
       from, to = nil
       if year
-        d = Date.strptime year, "%Y"
+        d    = Date.strptime year, "%Y"
         from = d.strftime "%m/%d/%Y"
         to   = d.next_year.prev_day.strftime "%m/%d/%Y"
       end
-      url  = "#{DOMAIN}/publications/search?keywords-lg=#{ref_nbr}"
+      url  = "#{DOMAIN}/publications/search?keywords-lg=#{text}"
       url += "&dateFrom-lg=#{from}" if from
       url += "&dateTo-lg=#{to}" if to
       url += if /PD/ =~ opts[:stage]
@@ -44,7 +91,7 @@ module RelatonNist
              end
 
       doc  = Nokogiri::HTML OpenURI.open_uri(::Addressable::URI.parse(url).normalize)
-      hits = doc.css("table.publications-table > tbody > tr").map do |h|
+      doc.css("table.publications-table > tbody > tr").map do |h|
         link  = h.at("td/div/strong/a")
         serie = h.at("td[1]").text.strip
         code  = h.at("td[2]").text.strip
@@ -59,39 +106,59 @@ module RelatonNist
           }, self
         )
       end
-      hits.sort! do |a, b|
-        if a.sort_value != b.sort_value
-          b.sort_value - a.sort_value
+    end
+
+    # Fetches data form json
+    # @param docid [String]
+    def from_json(docid, **opts)
+      data.select do |doc|
+        if year
+          d = Date.strptime year, "%Y"
+          idate = RelatonNist.parse_date doc["issued-date"]
+          next unless idate.between? d, d.next_year.prev_day
+        end
+        if /PD/ =~ opts[:stage]
+          next unless %w[draft-public draft-prelim].include? doc["status"]
         else
-          (b.hit[:release_date] - a.hit[:release_date]).to_i
+          next unless doc["status"] == "final"
         end
+        doc["docidentifier"] =~ Regexp.new(docid)
+      end.map do |h|
+        /(?<serie>(?<=-)\w+$)/ =~ h["series"]
+        title = [h["title-main"], h["title-sub"]].compact.join " - "
+        release_date = RelatonNist.parse_date h["published-date"]
+        Hit.new(
+          {
+            code: h["docidentifier"], serie: serie.upcase, title: title,
+            url: h["uri"], status: h["status"], release_date: release_date,
+            json: h
+          }, self
+        )
       end
-      concat hits
-      # concat(hits.map { |h| Hit.new(h, self) })
-      @fetched = false
-      # @hit_pages = hit_pages
     end
-    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
-    # @return [Iecbib::HitCollection]
-    def fetch
-      workers = RelatonBib::WorkersPool.new 4
-      workers.worker(&:fetch)
-      each do |hit|
-        workers << hit
+    # Fetches json data
+    # @return [Hash]
+    def data
+      ctime = File.ctime DATAFILE if File.exist? DATAFILE
+      if !ctime || ctime.to_date < Date.today
+        resp = OpenURI.open_uri("https://csrc.nist.gov/CSRC/media/feeds/metanorma/pubs-export.meta")
+        if !ctime || ctime < resp.last_modified
+          @data = nil
+          zip = OpenURI.open_uri "https://csrc.nist.gov/CSRC/media/feeds/metanorma/pubs-export.zip"
+          FileUtils.mv zip.path, DATAFILE
+        end
       end
-      workers.end
-      workers.result
-      @fetched = true
-      self
-    end
+      return if @data
 
-    def to_s
-      inspect
-    end
-
-    def inspect
-      "<#{self.class}:#{format('%#.14x', object_id << 1)} @fetched=#{@fetched}>"
+      Zip::File.open(DATAFILE) do |zf|
+        zf.each do |f|
+          @data = JSON.parse f.get_input_stream.read
+          break
+        end
+      end
+      @data
     end
+    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
   end
 end

data/lib/relaton_nist/nist_bibliography.rb

@@ -15,8 +15,7 @@ module RelatonNist
       def search(text, year = nil, opts = {})
         HitCollection.new text, year, opts
       rescue OpenURI::HTTPError, SocketError
-        warn "Could not access https://www.nist.gov"
-        []
+        raise RelatonBib::RequestError, "Could not access https://www.nist.gov"
       end
 
       # @param code [String] the NIST standard Code to look up (e..g "8200")
@@ -83,26 +82,22 @@ module RelatonNist
       # @retur [Hash]
       def nistbib_results_filter(result, year, opts)
         missed_years = []
+        iter = opts[:stage]&.slice(-3, 1)
+        iteration = case iter
+                    when "I" then "1"
+                    when "F" then "final"
+                    else iter
+                    end
         result.each_slice(3) do |s| # ISO website only allows 3 connections
           fetch_pages(s, 3).each_with_index do |r, _i|
             if opts[:issued_date]
-              r.dates.select { |d| d.type == "issued" }.each do |d|
-                next unless opts[:issued_date] == d.on
-              end
+              ids = r.dates.select { |d| d.type == "issued" && d.on == opts[:issued_date] }
+              next if ids.empty?
             elsif opts[:updated_date]
-              r.dates.select { |d| d.type == "published" }.each do |d|
-                next unless opts[:updated_date] == d.on
-              end
-            end
-            if opts[:stage]
-              iter = opts[:stage][-3]
-              iteration = case iter
-                          when "I" then 1
-                          when "F" then "final"
-                          else iter.to_i
-                          end
-              next if iter && r.status.iteration != iteration
+              pds = r.dates.select { |d| d.type == "published" && d.on == opts[:updated_date] }
+              next if pds.empty?
             end
+            next if iter && r.status.iteration != iteration
             return { ret: r } if !year
 
             r.dates.select { |d| d.type == "published" }.each do |d|

data/lib/relaton_nist/scrapper.rb

@@ -11,23 +11,55 @@ module RelatonNist
       # @param hit_data [Hash]
       # @return [Hash]
       def parse_page(hit_data)
-        doc = get_page hit_data[:url]
-
-        docid = fetch_docid(doc)
+        item_data = if hit_data[:json]
+                      from_json hit_data
+                    else
+                      from_csrs hit_data
+                    end
         doctype = "standard"
         titles = fetch_titles(hit_data)
-        unless /^(SP|NISTIR|FIPS) /.match docid[0].id
-          doctype = id_cleanup(docid[0].id)
-          docid[0] = RelatonBib::DocumentIdentifier.new(id: titles[0][:content], type: "NIST")
+        unless /^(SP|NISTIR|FIPS) / =~ item_data[:docid][0].id
+          doctype = id_cleanup(item_data[:docid][0].id)
+          item_data[:docid][0] = RelatonBib::DocumentIdentifier.new(
+            id: titles[0][:content], type: "NIST",
+          )
         end
+        item_data[:fetched] = Date.today.to_s
+        item_data[:type] = "standard"
+        item_data[:titles] = titles
+        item_data[:doctype] = doctype
+
+        NistBibliographicItem.new(**item_data)
+      end
+
+      private
+
+      def from_json(hit_data)
+        json = hit_data[:json]
+        {
+          link: fetch_link(json),
+          docid: fetch_docid(json["docidentifier"]),
+          dates: fetch_dates(json, hit_data[:release_date]),
+          contributors: fetch_contributors(json),
+          edition: fetch_edition(json),
+          language: [json["language"]],
+          script: [json["script"]],
+          # abstract: fetch_abstract(doc),
+          docstatus: fetch_status(json, hit_data[:status]),
+          copyright: fetch_copyright(json["published-date"]),
+          relations: fetch_relations_json(json),
+          # series: fetch_series(json),
+          keyword: fetch_keywords(json),
+          commentperiod: fetch_commentperiod_json(json),
+        }
+      end
 
-        NistBibliographicItem.new(
-          fetched: Date.today.to_s,
-          type: "standard",
+      def from_csrs(hit_data)
+        doc = get_page hit_data[:url]
+        {
           # id: fetch_id(doc),
-          titles: titles,
           link: fetch_link(doc),
-          docid: docid,
+          docid: fetch_docid(doc),
           dates: fetch_dates(doc, hit_data[:release_date]),
           contributors: fetch_contributors(doc),
           edition: fetch_edition(hit_data[:code]),
@@ -40,8 +72,7 @@ module RelatonNist
           series: fetch_series(doc),
           keyword: fetch_keywords(doc),
           commentperiod: fetch_commentperiod(doc),
-          doctype: doctype,
-        )
+        }
       end
       # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
@@ -52,8 +83,6 @@ module RelatonNist
         id.sub(/ \(WITHDRAWN\)/, "").sub(/ \(([^) ]+ )?DRAFT\)/i, "")
       end
 
-      private
-
       # Get page.
       # @param path [String] page's path
       # @return [Array<Nokogiri::HTML::Document, String>]
@@ -61,16 +90,23 @@ module RelatonNist
         uri = URI url
         resp = Net::HTTP.get_response(uri) # .encode("UTF-8")
         Nokogiri::HTML(resp.body)
+      rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError,
+             Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError
+        raise RelatonBib::RequestError, "Could not access #{url}"
       end
 
       # Fetch docid.
-      # @param doc [Nokogiri::HTML::Document]
+      # @param doc [Nokogiri::HTML::Document, String]
       # @return [Array<RelatonBib::DocumentIdentifier>]
       def fetch_docid(doc)
-        item_ref = doc.at("//div[contains(@class, 'publications-detail')]/h3").
-          text.strip
-        return [RelatonBib::DocumentIdentifier.new(type: "NIST", id: "?")] unless item_ref
-
+        item_ref = if doc.is_a? String
+                     doc
+                   else
+                     doc.at(
+                       "//div[contains(@class, 'publications-detail')]/h3",
+                     )&.text&.strip
+                   end
+        item_ref ||= "?"
         [RelatonBib::DocumentIdentifier.new(id: item_ref, type: "NIST")]
       end
 
@@ -83,56 +119,48 @@ module RelatonNist
       # end
 
       # Fetch status.
-      # @param doc [Nokogiri::HTML::Document]
+      # @param doc [Nokogiri::HTML::Document, Hash]
       # @param status [String]
-      # @return [Hash]
+      # @return [RelatonNist::DocumentStatus]
       def fetch_status(doc, status)
-        case status
-        when "draft (withdrawn)"
-          stage = "draft-public"
-          subst = "withdrawn"
-        when "retired draft"
-          stage = "draft-public"
-          subst = "retired"
-        when "withdrawn"
-          stage = "final"
-          subst = "withdrawn"
-        when "draft"
-          stage = "draft-public"
-          subst = "active"
+        if doc.is_a? Hash
+          stage = doc["status"]
+          subst = doc["substage"]
+          iter = doc["iteration"] == "initial" ? 1 : doc["iteration"]
         else
-          stage = status
-          subst = "active"
-        end
+          case status
+          when "draft (withdrawn)"
+            stage = "draft-public"
+            subst = "withdrawn"
+          when "retired draft"
+            stage = "draft-public"
+            subst = "retired"
+          when "withdrawn"
+            stage = "final"
+            subst = "withdrawn"
+          when "draft"
+            stage = "draft-public"
+            subst = "active"
+          else
+            stage = status
+            subst = "active"
+          end
+
+          iter = nil
+          if stage.include? "draft"
+            iter = 1
+            history = doc.xpath("//span[@id='pub-history-container']/a"\
+                                "|//span[@id='pub-history-container']/span")
+            history.each_with_index do |h, idx|
+              next if h.name == "a"
 
-        iter = nil
-        if stage.include? "draft"
-          iter = 1
-          history = doc.xpath("//span[@id='pub-history-container']/a"\
-                              "|//span[@id='pub-history-container']/span")
-          history.each_with_index do |h, idx|
-            next if h.name == "a"
-
-            iter = idx + 1 if idx.positive?
-            # iter = if lsif idx < (history.size - 1) && !history.last.text.include?("Draft")
-            #          "final"
-            #        elsif idx.positive? then idx + 1
-            #        end
-            break
+              iter = idx + 1 if idx.positive?
+              break
+            end
           end
         end
 
-        # if doc.at "//p/strong[text()='Withdrawn:']"
-        #   substage = "withdrawn"
-        # else
-        #   substage = "active"
-        #   item_ref = doc.at(
-        #     "//div[contains(@class, 'publications-detail')]/h3",
-        #   ).text.strip
-        #   wip = item_ref.match(/(?<=\()\w+/).to_s
-        #   stage = "draft-public" if wip == "DRAFT"
-        # end
-        RelatonNist::DocumentStatus.new stage: stage, substage: subst, iteration: iter
+        RelatonNist::DocumentStatus.new stage: stage, substage: subst, iteration: iter.to_s
       end
 
       # Fetch titles.
@@ -144,46 +172,87 @@ module RelatonNist
 
       # Fetch dates
       # @param doc [Nokogiri::HTML::Document]
+      # @param release_date [Date]
       # @return [Array<Hash>]
       def fetch_dates(doc, release_date)
         dates = [{ type: "published", on: release_date.to_s }]
 
-        d = doc.at("//span[@id='pub-release-date']").text.strip
-        date = if /(?<date>\w+\s\d{4})/ =~ d
-                 Date.strptime(date, "%B %Y")
-               elsif /(?<date>\w+\s\d{1,2},\s\d{4})/ =~ d
-                 Date.strptime(date, "%B %d, %Y")
-               end
-        dates << { type: "issued", on: date.to_s }
-
+        if doc.is_a? Hash
+          issued = RelatonNist.parse_date doc["issued-date"]
+          updated = RelatonNist.parse_date doc["updated-date"]
+          dates << { type: "updated", on: updated.to_s } if updated
+          obsoleted = RelatonNist.parse_date doc["obsoleted-date"]
+          dates << { type: "obsoleted", on: obsoleted.to_s } if obsoleted
+        else
+          d = doc.at("//span[@id='pub-release-date']").text.strip
+          issued = RelatonNist.parse_date d
+        end
+        dates << { type: "issued", on: issued.to_s }
         dates
       end
 
+      # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      # @param doc [Nokogiri::HTML::Document, Hash]
+      # @return [Array<RelatonBib::ContributionInfo>]
       def fetch_contributors(doc)
-        name = "National Institute of Standards and Technology"
-        org = RelatonBib::Organization.new(
-          name: name, url: "www.nist.gov", abbreviation: "NIST",
-        )
-        contribs = [
-          RelatonBib::ContributionInfo.new(entity: org, role: ["publisher"]),
-        ]
-
-        authors = doc.at('//h4[.="Author(s)"]/following-sibling::p')
-        contribs += contributors(authors, "author")
+        contribs = []
+        if doc.is_a? Hash
+          contribs += contributors_json(
+            doc["authors"], "author", doc["language"], doc["script"]
+          )
+          contribs + contributors_json(
+            doc["editors"], "editor", doc["language"], doc["script"]
+          )
+        else
+          name = "National Institute of Standards and Technology"
+          org = RelatonBib::Organization.new(
+            name: name, url: "www.nist.gov", abbreviation: "NIST",
+          )
+          contribs << RelatonBib::ContributionInfo.new(entity: org, role: ["publisher"])
+          authors = doc.at('//h4[.="Author(s)"]/following-sibling::p')
+          contribs += contributors(authors, "author")
+          editors = doc.at('//h4[.="Editor(s)"]/following-sibling::p')
+          contribs + contributors(editors, "editor")
+        end
+      end
 
-        editors = doc.at('//h4[.="Editor(s)"]/following-sibling::p')
-        contribs + contributors(editors, "editor")
+      # @param doc [Array<Hash>]
+      # @param role [String]
+      # @return [Array<RelatonBib::ContributionInfo>]
+      def contributors_json(doc, role, lang = "en", script = "Latn")
+        doc.map do |contr|
+          if contr["affiliation"]
+            if contr["affiliation"]["acronym"]
+              abbrev = RelatonBib::LocalizedString.new(contr["affiliation"]["acronym"])
+            end
+            org = RelatonBib::Organization.new(
+              name: contr["affiliation"]["name"], abbreviation: abbrev,
+            )
+          end
+          if contr["surname"]
+            affiliation = RelatonBib::Affilation.new org
+            entity = RelatonBib::Person.new(
+              name: full_name(contr, lang, script), affiliation: [affiliation],
+            )
+          else
+            entity = org
+          end
+          RelatonBib::ContributionInfo.new entity: entity, role: [role]
+        end
       end
 
       # rubocop:disable Metrics/CyclomaticComplexity
-      def contributors(doc, role)
+      # @param doc [Nokogiri::HTML::Element, Array<Hash>]
+      # @param role [String]
+      # @return [Array<RelatonBib::ContributionInfo>]
+      def contributors(doc, role, lang = "en", script = "Latn")
         return [] if doc.nil?
 
         doc.text.split(", ").map do |contr|
           /(?<an>.+?)(\s+\((?<abbrev>.+?)\))?$/ =~ contr
           if abbrev && an.downcase !~ /(task|force|group)/ && an.split.size.between?(2, 3)
             fullname = RelatonBib::FullName.new(
-              completename: RelatonBib::LocalizedString.new(an, "en", "Latn"),
+              completename: RelatonBib::LocalizedString.new(an, lang, script),
             )
             case abbrev
             when "NIST"
@@ -199,7 +268,7 @@ module RelatonNist
             org = RelatonBib::Organization.new name: org_name, url: url, abbreviation: abbrev
             affiliation = RelatonBib::Affilation.new org
             entity = RelatonBib::Person.new(
-              name: fullname, affiliation: [affiliation], contacts: [],
+              name: fullname, affiliation: [affiliation],
             )
           else
             entity = RelatonBib::Organization.new name: an, abbreviation: abbrev
@@ -207,17 +276,49 @@ module RelatonNist
           RelatonBib::ContributionInfo.new entity: entity, role: [role]
         end
       end
-      # rubocop:enable Metrics/CyclomaticComplexity
+      # rubocop:enable Metrics/CyclomaticComplexity, Metrics/AbcSize, Metrics/MethodLength
+
+      # @param name [Hash]
+      # @param lang [Strong]
+      # @param script [String]
+      # @return [RelatonBib::FullName]
+      def full_name(name, lang, script)
+        RelatonBib::FullName.new(
+          surname: RelatonBib::LocalizedString.new(name["surname"], lang, script),
+          forenames: name_parts(name["givenName"], lang, script),
+          additions: name_parts(name["suffix"], lang, script),
+          prefix: name_parts(name["title"], lang, script),
+          completename: RelatonBib::LocalizedString.new(name["fullName"], lang, script),
+        )
+      end
 
-      def fetch_edition(code)
-        return unless /(?<=Rev\.\s)(?<rev>\d+)/ =~ code
+      # @param part [String, NilClass]
+      # @param lang [Strong]
+      # @param script [String]
+      # @return [Array<RelatonBib::LocalizedString>]
+      def name_parts(part, lang, script)
+        return [] unless part
+
+        [RelatonBib::LocalizedString.new(name[part], lang, script)]
+      end
+
+      # @param doc [String, Hash]
+      # @return [String, NilClass]
+      def fetch_edition(doc)
+        if doc.is_a? Hash
+          return unless doc["edition"]
+
+          rev = doc["edition"]
+        else
+          return unless /(?<=Rev\.\s)(?<rev>\d+)/ =~ doc
+        end
 
         "Revision #{rev}"
       end
 
       # Fetch abstracts.
       # @param doc [Nokigiri::HTML::Document]
-      # @return [Array<Array>]
+      # @return [Array<Hash>]
       def fetch_abstract(doc)
         abstract_content = doc.xpath('//div[contains(@class, "pub-abstract-callout")]/div[1]/p').text
         [{
@@ -229,58 +330,82 @@ module RelatonNist
       end
 
       # Fetch copyright.
-      # @param title [String]
+      # @param doc [Nokogiri::HTL::Document, String]
       # @return [Hash]
       def fetch_copyright(doc)
         name = "National Institute of Standards and Technology"
         url = "www.nist.gov"
-        d = doc.at("//span[@id='pub-release-date']").text.strip
+        d = if doc.is_a? String then doc
+            else
+              doc.at("//span[@id='pub-release-date']").text.strip
+            end
         from = d.match(/\d{4}/).to_s
         { owner: { name: name, abbreviation: "NIST", url: url }, from: from }
       end
 
       # Fetch links.
-      # @param doc [Nokogiri::HTML::Document]
+      # @param doc [Nokogiri::HTML::Document, Hash]
       # @return [Array<Hash>]
       def fetch_link(doc)
-        pub = doc.at "//p/strong[.='Publication:']"
         links = []
-        pdf = pub.at "./following-sibling::a[.=' Local Download']"
-        links << { type: "pdf", content: pdf[:href] } if pdf
-        doi = pub.at("./following-sibling::a[contains(.,'(DOI)')]")
-        links << { type: "doi", content: doi[:href] } if doi
+        if doc.is_a? Hash
+          links << { type: "uri", content: doc["uri"] } if doc["uri"]
+          doi = "https://doi.org/" + doc["doi"] if doc["doi"]
+        else
+          pub = doc.at "//p/strong[.='Publication:']"
+          pdf = pub.at "./following-sibling::a[.=' Local Download']"
+          doi = pub.at("./following-sibling::a[contains(.,'(DOI)')]")&.attr :href
+          links << { type: "pdf", content: pdf[:href] } if pdf
+        end
+        links << { type: "doi", content: doi } if doi
         links
       end
 
       # Fetch relations.
       # @param doc [Nokogiri::HTML::Document]
-      # @return [Array<Hash>]
+      # @return [Array<RelatonBib::DocumentRelation>]
       def fetch_relations(doc)
         relations = doc.xpath('//span[@id="pub-supersedes-container"]/a').map do |r|
-          doc_relation "supersedes", r
+          doc_relation "supersedes", r.text, DOMAIN + r[:href]
         end
 
         relations += doc.xpath('//span[@id="pub-part-container"]/a').map do |r|
-          doc_relation "partOf", r
+          doc_relation "partOf", r.text, DOMAIN + r[:href]
         end
 
         relations + doc.xpath('//span[@id="pub-related-container"]/a').map do |r|
-          doc_relation "updates", r
+          doc_relation "updates", r.text, DOMAIN + r[:href]
         end
       end
 
-      def doc_relation(type, ref)
+      def fetch_relations_json(doc)
+        relations = doc["supersedes"].map do |r|
+          doc_relation "supersedes", r["docidentifier"], r["uri"]
+        end
+
+        relations + doc["superseded-by"].map do |r|
+          doc_relation "updates", r["docidentifier"], r["uri"]
+        end
+      end
+
+      # @param type [String]
+      # @param ref [String]
+      # @param uri [String]
+      # @return [RelatonBib::DocumentRelation]
+      def doc_relation(type, ref, uri, lang = "en", script = "Latn")
         RelatonBib::DocumentRelation.new(
           type: type,
           bibitem: RelatonBib::BibliographicItem.new(
             formattedref: RelatonBib::FormattedRef.new(
-              content: ref.text, language: "en", script: "Latn", format: "text/plain",
+              content: ref, language: lang, script: script, format: "text/plain",
             ),
-            link: [RelatonBib::TypedUri.new(type: "src", content: DOMAIN + ref[:href])],
+            link: [RelatonBib::TypedUri.new(type: "src", content: uri)],
           ),
         )
       end
 
+      # @param doc [Nokogiri::HTML::Document]
+      # @return [Array<RelatonBib::Series>]
       def fetch_series(doc)
         series = doc.xpath "//span[@id='pub-history-container']/a"\
           "|//span[@id='pub-history-container']/span"
@@ -305,11 +430,19 @@ module RelatonNist
         end.select { |s| s }
       end
 
+      # @param doc [Nokogiri::HTML::Document, Hash]
+      # @return [Array<RelatonNist::Keyword>]
       def fetch_keywords(doc)
-        kws = doc.xpath "//span[@id='pub-keywords-container']/span"
-        kws.map { |kw| Keyword.new kw.text }
+        kws = if doc.is_a? Hash
+                doc["keywords"]
+              else
+                doc.xpath "//span[@id='pub-keywords-container']/span"
+              end
+        kws.map { |kw| Keyword.new kw.is_a?(String) ? kw : kw.text }
       end
 
+      # @param doc [Nokogiri::HTML::Document]
+      # @return [RelatonNist::CommentPeriod, NilClass]
       def fetch_commentperiod(doc)
         cp = doc.at "//span[@id='pub-comments-due']"
         return unless cp
@@ -324,6 +457,12 @@ module RelatonNist
         extended = ext.empty? ? nil : Date.strptime(ext, "%B %d, %Y")
         CommentPeriod.new from, to, extended
       end
+
+      # @param json [Hash]
+      # @return [RelatonNist::CommentPeriod, NilClass]
+      def fetch_commentperiod_json(json)
+        CommentPeriod.new json["comment-from"], json["comment-to"] if json["comment-from"]
+      end
     end
   end
 end

data/lib/relaton_nist/version.rb

@@ -1,3 +1,3 @@
 module RelatonNist
-  VERSION = "0.2.1".freeze
+  VERSION = "0.2.2".freeze
 end

data/relaton_nist.gemspec

@@ -36,4 +36,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "webmock"
 
   spec.add_dependency "relaton-bib", "~> 0.2.0"
+  spec.add_dependency "rubyzip"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-nist
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-06-19 00:00:00.000000000 Z
+date: 2019-06-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -178,6 +178,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.2.0
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 'RelatonNist: retrive NIST standards.'
 email:
 - open.source@ribose.com
@@ -200,6 +214,7 @@ files:
 - lib/relaton/processor.rb
 - lib/relaton_nist.rb
 - lib/relaton_nist/comment_period.rb
+- lib/relaton_nist/data/pubs-export.zip
 - lib/relaton_nist/document_status.rb
 - lib/relaton_nist/hit.rb
 - lib/relaton_nist/hit_collection.rb
@@ -230,7 +245,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 2.6.12
 signing_key: 
 specification_version: 4
 summary: 'RelatonNist: retrive NIST standards.'