relaton-nist 1.12.1 → 1.12.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f664ac25e37f4cc366dbce555dd02629e6aba762d7d5305ccfdf7824035bf326
-  data.tar.gz: ed762c28f2f693b08cffe28f1b0f01ba7765bcbc6a3ab24432265a4d18faafee
+  metadata.gz: a08c33ae20e118a973b19b9697114dd4be09331bc3322a3d75250959c4cb0880
+  data.tar.gz: 1066aab4db2167b2551fd59a76248a96bbcfddbbcc11e0894cb16d91af0fe69c
 SHA512:
-  metadata.gz: '095a99fbc11c08d2cc968a181444eb3913a01ef321477b13eb43e5f68949b378c494b89620c5dee6bf409c0f36e8246c55bad27afb3f3609165b24694f4fe5f2'
-  data.tar.gz: ba5172f3df58f5830fa4aa6e3c90ed250de30ac71206f038b8fbc867aa5d2cc9a9f1a6bb8463684da2a2ddbdf9032232aaa9ba20930c8460d0c8c36289c281df
+  metadata.gz: e0b46445fda68e042630f4eccfc925a317dbe22a3331fe5ef86eebcceb7a036134bedba1a2b6679987d4a2b727ee2590887d52a26c014adbbd27b382b70b2adc
+  data.tar.gz: cdc10291398c3f037a00f7a2b013f3a159ed1b8803a0d401522cfb4df0119036cd7b1a9cbfb28ebdf05819a4f365fd4bc0d284fc3031ef34311418e279e35eba

data/README.adoc

@@ -1,4 +1,4 @@
-= RelatonNist: retrieve NIST Standards for bibliographic use using the BibliographicItem model
+= Relaton for NIST: bibliographic retrieval of NIST publications
 
 image:https://img.shields.io/gem/v/relaton-nist.svg["Gem Version", link="https://rubygems.org/gems/relaton-nist"]
 image:https://github.com/relaton/relaton-nist/workflows/macos/badge.svg["Build Status (macOS)", link="https://github.com/relaton/relaton-nist/actions?workflow=macos"]
@@ -8,10 +8,41 @@ image:https://codeclimate.com/github/relaton/relaton-nist/badges/gpa.svg["Code C
 image:https://img.shields.io/github/issues-pr-raw/relaton/relaton-nist.svg["Pull Requests", link="https://github.com/relaton/relaton-nist/pulls"]
 image:https://img.shields.io/github/commits-since/relaton/relaton-nist/latest.svg["Commits since latest",link="https://github.com/relaton/relaton-nist/releases"]
 
+== Purpose
 
-RelatonNist is a Ruby gem that implements the https://github.com/metanorma/metanorma-model-iso#iso-bibliographic-item[IsoBibliographicItem model].
+`relaton-nist` provides bibliographic information of NIST publications using the
+https://github.com/metanorma/metanorma-model-nist#nist-bibliographic-item-model[NistBibliographicItem model].
+
+Relaton for NIST has been developed in cooperation with the NIST Cybersecurity
+Resource Center (CSRC) and the Computer Security Division (ITL/CSD).
+
+== Data sources
+
+Relaton for NIST retrieves bibliographic information from two sources:
+
+* bibliographic feed from the NIST Cybersecurity Resource Center (CSRC) of
+all CSRC publications (in Relaton JSON)
+* bibliographic dataset from the NIST Library through the Information Services
+Office (ISO) that contains information about all NIST Technical Publications
+(https://github.com/usnistgov/NIST-Tech-Pubs[GitHub])
+
+Bibliographic information offered through CSRC are provided with enhanced
+metadata that is not available at the NIST Library dataset, including:
+
+* public drafts (the NIST Library dataset only contains final publications)
+* revision information: revision number, iteration
+* document stage information: retired, withdrawn, etc.
+* bibliographic dates, including issued date, updated date, published date,
+  obsolete date, commenting period
+* document relationships: supersession and replacements
+* contacts: enhanced name parts and affiliation information
+
+Relaton for NIST therefore uses the following order of priority for the data
+sources:
+
+. bibliographic feed from NIST CSRC
+. NIST Library dataset
 
-You can use it to retrieve metadata of NIST Standards from https://csrc.nist.gov, and access such metadata through the `IsoBibliographicItem` object.
 
 == Installation
 

data/lib/relaton_nist/data_fetcher.rb

@@ -217,7 +217,7 @@ module RelatonNist
       prf, srs, num = pub_id(doc).split
       sname = series[srs] || srs
       title = RelatonBib::TypedTitleString.new(content: "#{prf} #{sname}")
-      abbr = RelatonBib::LocalizedString.new("#{prf} #{srs}")
+      abbr = RelatonBib::LocalizedString.new srs
       [RelatonBib::Series.new(title: title, abbreviation: abbr, number: num)]
     end
 

data/lib/relaton_nist/hit.rb

@@ -14,7 +14,7 @@ module RelatonNist
     # @return [Iteger]
     def sort_value # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
       @sort_value ||= begin
-        sort_phrase = [hit[:serie], hit[:code], hit[:title]].join " "
+        sort_phrase = [hit[:series], hit[:code], hit[:title]].join " "
         corr = hit_collection&.text&.split&.map do |w|
           if w =~ /\w+/ &&
               sort_phrase =~ Regexp.new(Regexp.escape(w), Regexp::IGNORECASE)

data/lib/relaton_nist/hit_collection.rb

@@ -25,8 +25,75 @@ module RelatonNist
       sort_hits!
     end
 
+    # @return [Array<RelatonNist::Hit>]
+    def search_filter # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
+      @array.select do |item|
+        %r{
+          ^(?:(?:NIST|NBS)\s?)?
+          (?:(?<series>(?:SP|FIPS|IR|ITL\sBulletin|White\sPaper))\s)?
+          (?<code>[0-9-]{3,}[A-Z]?)
+          (?<prt1>pt\d+)?
+          (?<vol1>v\d+)?
+          (?<ver1>ver[\d.]+)?
+          (?<rev1>r\d+)?
+          (?:\s(?<prt2>Part\s\d+))?
+          (?:\s(?<vol2>Vol\.\s\d+))?
+          (?:\s(?<ver2>(?:Ver\.|Version)\s[\d.]+))?
+          (?:\s(?<rev2>Rev\.\s\d+))?
+          (?:\s(?<add>Add)endum)?
+        }x =~ item.hit[:code]
+        (refparts[:code] && [series, item.hit[:series]].include?(refparts[:series]) && refparts[:code] == code &&
+          long_to_short(refparts[:prt1], refparts[:prt2]) == long_to_short(prt1, prt2) &&
+          long_to_short(refparts[:vol1], refparts[:vol2]) == long_to_short(vol1, vol2) &&
+          long_to_short(refparts[:ver1], refparts[:ver2]) == long_to_short(ver1, ver2) &&
+          long_to_short(refparts[:rev1], refparts[:rev2]) == long_to_short(rev1, rev2) &&
+          long_to_short(refparts[:add1], refparts[:add2]) == add) || item.hit[:title]&.include?(text.sub(/^NIST\s/, ""))
+      end
+    end
+
     private
 
+    def refparts # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      @refparts ||= {
+        perfix: match(/^(NIST|NBS)\s?/, text),
+        series: match(/(SP|FIPS|IR|ITL\sBulletin|White\sPaper)(?=\.|\s)/, text),
+        code: match(/(?<=\.|\s)[0-9-]{3,}[A-Z]?/, text),
+        prt1: match(/(?<=(\.))?pt(?(1)-)[A-Z\d]+/, text),
+        vol1: match(/(?<=(\.))?v(?(1)-)\d+/, text),
+        ver1: match(/(?<=(\.))?ver(?(1)[-\d]|[.\d])+/, text)&.gsub(/-/, "."),
+        rev1: match(/(?<=[^a-z])(?<=(\.))?r(?(1)-)\d+/, text),
+        add1: match(/(?<=(\.))?add(?(1)-)\d+/, text),
+        prt2: match(/(?<=\s)Part\s[A-Z\d]+/, text),
+        vol2: match(/(?<=\s)Vol\.\s\d+/, text),
+        ver2: match(/(?<=\s)Ver\.\s\d+/, text),
+        rev2: match(/(?<=\s)Rev\.\s\d+/, text),
+        add2: match(/(?<=\/)Add/, text),
+      }
+    end
+
+    def match(regex, code)
+      regex.match(code)&.to_s
+    end
+
+    def full_ref # rubocop:disable Metrics/AbcSize
+      @full_ref ||= begin
+        ref = "#{refparts[:perfix]}#{refparts[:series]} #{refparts[:code]}"
+        ref += long_to_short(refparts[:prt1], refparts[:prt2]).to_s
+        ref += long_to_short(refparts[:vol1], refparts[:vol2]).to_s
+        ref
+      end
+    end
+
+    # @param short [String]
+    # @param long [String]
+    # @return [String, nil]
+    def long_to_short(short, long)
+      return short.sub(/-/, "") if short
+      return unless long
+
+      long.sub(/Part\s/, "pt").sub(/Vol\.\s/, "v").sub(/Rev\.\s/, "r").sub(/(Ver\.|Version)\s/, "ver")
+    end
+
     def sort_hits!
       @array.sort! do |a, b|
         if a.sort_value == b.sort_value
@@ -39,7 +106,8 @@ module RelatonNist
     end
 
     def from_ga # rubocop:disable Metrics/AbcSize
-      fn = text.gsub(%r{[/\s:.]}, "_").upcase
+      ref = full_ref
+      fn = ref.gsub(%r{[/\s:.]}, "_").upcase
       yaml = OpenURI.open_uri "#{GHNISTDATA}#{fn}.yaml"
       hash = YAML.safe_load yaml
       bib = RelatonNist::NistBibliographicItem.from_hash hash
@@ -52,55 +120,15 @@ module RelatonNist
       raise e
     end
 
-    # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-
-    # @param stage [String]
-    # @return [Array<RelatonNist::Hit>]
-    # def from_csrc(**opts)
-    #   from, to = nil
-    #   if year
-    #     d    = Date.strptime year, "%Y"
-    #     from = d.strftime "%m/%d/%Y"
-    #     to   = d.next_year.prev_day.strftime "%m/%d/%Y"
-    #   end
-    #   url = "#{DOMAIN}/publications/search?keywords-lg=#{text}"\
-    #         "&sortBy-lg=relevence"
-    #   url += "&dateFrom-lg=#{from}" if from
-    #   url += "&dateTo-lg=#{to}" if to
-    #   url += if /PD/.match? opts[:stage]
-    #            "&status-lg=Draft,Retired Draft,Withdrawn"
-    #          else
-    #            "&status-lg=Final,Withdrawn"
-    #          end
-
-    #   doc = Nokogiri::HTML OpenURI.open_uri(::Addressable::URI.parse(url).normalize)
-    #   doc.css("table.publications-table > tbody > tr").map do |h|
-    #     link  = h.at("td/div/strong/a")
-    #     serie = h.at("td[1]").text.strip
-    #     code  = h.at("td[2]").text.strip
-    #     title = link.text
-    #     doc_url = DOMAIN + link[:href]
-    #     status = h.at("td[4]").text.strip.downcase
-    #     release_date = Date.strptime h.at("td[5]").text.strip, "%m/%d/%Y"
-    #     Hit.new(
-    #       {
-    #         code: code, serie: serie, title: title, url: doc_url,
-    #         status: status, release_date: release_date
-    #       }, self
-    #     )
-    #   end
-    # end
-    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
-
     # Fetches data form json
     # @param stage [String]
     # @return [Array<RelatonNist::Hit>]
     def from_json(**opts)
       select_data(**opts).map do |h|
-        /(?<serie>(?<=-)\w+$)/ =~ h["series"]
+        /(?<series>(?<=-)\w+$)/ =~ h["series"]
         title = [h["title-main"], h["title-sub"]].compact.join " - "
         release_date = RelatonBib.parse_date h["published-date"], false
-        Hit.new({ code: h["docidentifier"], serie: serie.upcase, title: title,
+        Hit.new({ code: h["docidentifier"], series: series.upcase, title: title,
                   url: h["uri"], status: h["status"],
                   release_date: release_date, json: h }, self)
       end
@@ -109,6 +137,7 @@ module RelatonNist
     # @param stage [String]
     # @return [Array<Hach>]
     def select_data(**opts) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength,Metrics/PerceivedComplexity
+      ref = "#{refparts[:series]} #{refparts[:code]}"
       d = Date.strptime year, "%Y" if year
       statuses = %w[draft-public draft-prelim]
       data.select do |doc|
@@ -119,7 +148,7 @@ module RelatonNist
         else
           next unless doc["status"] == "final"
         end
-        doc["docidentifier"].include? text
+        doc["docidentifier"].include?(ref) || doc["docidentifier"].include?(full_ref)
       end
     end
 
@@ -147,7 +176,6 @@ module RelatonNist
     #
     # @prarm ctime [Time, NilClass]
     def fetch_data(ctime)
-      # resp = OpenURI.open_uri("#{PUBS_EXPORT}.meta")
       if !ctime || ctime < OpenURI.open_uri("#{PUBS_EXPORT}.meta").last_modified
         @data = nil
         uri_open = URI.method(:open) || Kernel.method(:open)

data/lib/relaton_nist/nist_bibliography.rb

@@ -14,7 +14,8 @@ module RelatonNist
       # @param text [String]
       # @return [RelatonNist::HitCollection]
       def search(text, year = nil, opts = {})
-        HitCollection.search text, year, opts
+        ref = text.sub(/^NIST\sIR/, "NISTIR")
+        HitCollection.search ref, year, opts
       rescue OpenURI::HTTPError, SocketError, OpenSSL::SSL::SSLError => e
         raise RelatonBib::RequestError, e.message
       end
@@ -55,12 +56,12 @@ module RelatonNist
         end
 
         code += "-1" if opts[:all_parts]
-        nistbib_get1(code, year, opts)
+        nistbib_get(code, year, opts)
       end
 
       private
 
-      def nistbib_get1(code, year, opts)
+      def nistbib_get(code, year, opts)
         result = nistbib_search_filter(code, year, opts) || (return nil)
         ret = nistbib_results_filter(result, year, opts)
         if ret[:ret]
@@ -84,7 +85,7 @@ module RelatonNist
       # @option opts [String] :stage
       #
       # @return [Hash]
-      def nistbib_results_filter(result, year, opts)
+      def nistbib_results_filter(result, year, opts) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
         missed_years = []
         iter = /\w+(?=PD)|(?<=PD-)\w+/.match(opts[:stage])&.to_s
         iteration = case iter
@@ -133,111 +134,23 @@ module RelatonNist
       # @param year [String, nil]
       # @param opts [Hash]
       # @return [RelatonNist::HitCollection]
-      def nistbib_search_filter(code, year, opts) # rubocop:disable Metrics/MethodLength
+      def nistbib_search_filter(code, year, opts)
         warn "[relaton-nist] (\"#{code}\") fetching..."
-        # match = %r{
-        #   ^((?:NIST)\s)?
-        #   (?<serie>(SP|FIPS|NISTIR|ITL\sBulletin|White\sPaper))\s
-        #   (?<code>[0-9-]{3,}[A-Z]?)
-        #   (?<prt1>pt\d+)?
-        #   (?<vol1>v\d+)?
-        #   (?<ver1>ver[\d\.]+)?
-        #   (?<rev1>r\d+)?
-        #   (\s(?<prt2>Part\s\d+))?
-        #   (\s(?<vol2>Vol\.\s\d+))?
-        #   (\s(?<ver2>(Ver\.|Version)\s[\d\.]+))?
-        #   (\s(?<rev2>Rev\.\s\d+))?
-        #   (\/(?<upd>Add))?
-        # }x.match(code)
-        # match ||= %r{
-        #   ^NIST\.
-        #   (?<serie>(SP|FIPS|IR|ITL\sBulletin|White\sPaper))\.
-        #   ((PD-\d+|PUB)\.)?
-        #   (?<code>[0-9-]{3,}[A-Z]?)
-        #   (\.(?<prt1>pt-\d+))?
-        #   (\.(?<vol1>v-\d+))?
-        #   (\.(?<ver1>ver-[\d\.]+))?
-        #   (\.(?<rev1>r-\d+))?
-        # }x.match(code)
-        matches = {
-          serie: match(/(SP|FIPS|(NIST)?IR|ITL\sBulletin|White\sPaper)(?=\.|\s)/, code),
-          code: match(/(?<=\.|\s)[0-9-]{3,}[A-Z]?/, code),
-          prt1: match(/(?<=(\.))?pt(?(1)-)[A-Z\d]+/, code),
-          vol1: match(/(?<=(\.))?v(?(1)-)\d+/, code),
-          ver1: match(/(?<=(\.))?ver(?(1)[-\d]|[\.\d])+/, code)&.gsub(/-/, "."),
-          rev1: match(/(?<=[^a-z])(?<=(\.))?r(?(1)-)\d+/, code),
-          add1: match(/(?<=(\.))?add(?(1)-)\d+/, code),
-          prt2: match(/(?<=\s)Part\s[A-Z\d]+/, code),
-          vol2: match(/(?<=\s)Vol\.\s\d+/, code),
-          ver2: match(/(?<=\s)Ver\.\s\d+/, code),
-          rev2: match(/(?<=\s)Rev\.\s\d+/, code),
-          add2: match(/(?<=\/)Add/, code),
-        }
-        ref = matches[:code] ? "#{matches[:serie]} #{matches[:code]}" : code
-        result = search(ref, year, opts)
-        selected_result = result.select { |i| search_filter i, matches, code }
-        return selected_result if selected_result.any? || !matches[:code]
-
-        search full_ref(matches)
-      end
-
-      def full_ref(matches)
-        ref = "#{matches[:serie]} #{matches[:code]}"
-        ref += long_to_short(matches[:prt1], matches[:prt2]).to_s
-        ref += long_to_short(matches[:vol1], matches[:vol2]).to_s
-        ref
-      end
-
-      def match(regex, code)
-        regex.match(code)&.to_s
-      end
-
-      # @param item [RelatonNist::Hit]
-      # @param matches [Hash]
-      # @param text [String]
-      # @return [Boolean]
-      def search_filter(item, matches, text) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
-        %r{
-          ^((?:NIST)\s)?
-          ((?<serie>(SP|FIPS|NISTIR|ITL\sBulletin|White\sPaper))\s)?
-          (?<code>[0-9-]{3,}[A-Z]?)
-          (?<prt1>pt\d+)?
-          (?<vol1>v\d+)?
-          (?<ver1>ver[\d.]+)?
-          (?<rev1>r\d+)?
-          (\s(?<prt2>Part\s\d+))?
-          (\s(?<vol2>Vol\.\s\d+))?
-          (\s(?<ver2>(Ver\.|Version)\s[\d.]+))?
-          (\s(?<rev2>Rev\.\s\d+))?
-          (\s(?<add>Add)endum)?
-        }x =~ item.hit[:code]
-        matches[:code] && [serie, item.hit[:serie]].include?(matches[:serie]) && matches[:code] == code &&
-          long_to_short(matches[:prt1], matches[:prt2]) == long_to_short(prt1, prt2) &&
-          long_to_short(matches[:vol1], matches[:vol2]) == long_to_short(vol1, vol2) &&
-          long_to_short(matches[:ver1], matches[:ver2]) == long_to_short(ver1, ver2) &&
-          long_to_short(matches[:rev1], matches[:rev2]) == long_to_short(rev1, rev2) &&
-          long_to_short(matches[:add1], matches[:add2]) == add || item.hit[:title].include?(text.sub(/^NIST\s/, ""))
-      end
-
-      # @param short [String]
-      # @param long [String]
-      # @return [String, nil]
-      def long_to_short(short, long)
-        return short.sub(/-/, "") if short
-        return unless long
-
-        long.sub(/Part\s/, "pt").sub(/Vol\.\s/, "v").sub(/Rev\.\s/, "r").sub(/(Ver\.|Version)\s/, "ver")
+        result = search(code, year, opts)
+        result.search_filter
       end
 
-      def fetch_ref_err(code, year, missed_years)
+      def fetch_ref_err(code, year, missed_years) # rubocop:disable Metrics/MethodLength
         id = year ? "#{code}:#{year}" : code
-        warn "[relaton-nist] WARNING: no match found online for #{id}. "\
-          "The code must be exactly like it is on the standards website."
-        warn "[relaton-nist] (There was no match for #{year}, though there were matches "\
-          "found for #{missed_years.join(', ')}.)" unless missed_years.empty?
-        if /\d-\d/ =~ code
-          warn "[relaton-nist] The provided document part may not exist, "\
-            "or the document may no longer be published in parts."
+        warn "[relaton-nist] WARNING: no match found online for #{id}. " \
+             "The code must be exactly like it is on the standards website."
+        unless missed_years.empty?
+          warn "[relaton-nist] (There was no match for #{year}, though there " \
+               "were matches found for #{missed_years.join(', ')}.)"
+        end
+        if /\d-\d/.match? code
+          warn "[relaton-nist] The provided document part may not exist, " \
+               "or the document may no longer be published in parts."
         end
         nil
       end

data/lib/relaton_nist/processor.rb

@@ -5,7 +5,7 @@ module RelatonNist
     def initialize # rubocop:disable Lint/MissingSuper
       @short = :relaton_nist
       @prefix = "NIST"
-      @defaultprefix = %r{^(NIST|NISTGCR|ITL Bulletin|JPCRD|NISTIR|CSRC|FIPS)(/[^\s])?\s}
+      @defaultprefix = %r{^(NIST|NBS|NISTGCR|ITL Bulletin|JPCRD|NISTIR|CSRC|FIPS)(/[^\s])?\s}
       @idtype = "NIST"
       @datasets = %w[nist-tech-pubs]
     end

data/lib/relaton_nist/version.rb

@@ -1,3 +1,3 @@
 module RelatonNist
-  VERSION = "1.12.1".freeze
+  VERSION = "1.12.4".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-nist
 version: !ruby/object:Gem::Version
-  version: 1.12.1
+  version: 1.12.4
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-07-01 00:00:00.000000000 Z
+date: 2022-07-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: equivalent-xml