relaton-nist 1.16.3 → 1.16.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b00f5d1fb998afa9409df9108f6336cf067fc41faef13e8aca3a95cc8b289596
-  data.tar.gz: 17f09df20a20798c1a56b5b7b887d7be9a8509e3a948b731603cbd95e684346e
+  metadata.gz: 9584f82c13f12da662266b2de0e11eb37241a6b3c4994d28720af44bdb713632
+  data.tar.gz: 1d14d47f043644e96cd42a8419117e12111623e5e80a7a2cb37ac6c1420d37c1
 SHA512:
-  metadata.gz: 1dfc55cfa95ad85bf38a3a582d5d92fea954f7fd7e66537d59d78f672efd798f10bb1bed8ee804c4c9b062017fa7202edd38295b9456c990368883bb77861436
-  data.tar.gz: 0e3b65872354a4c48dfd1f752ec160d3d2bdd47bafa9e678a3a5e200189589ce1e09b25e17df7c05bf6f643290f10ad3925bd9efb415525cf1c91894d3c6f92a
+  metadata.gz: 1d86cd19ea38a1743044ce99e28c6c5c0adc4805d5c45d6145ff4f724b10a9f9115a495cd226514fbd66e2213dce3ba807baddf3275528ae438917c36f3784a8
+  data.tar.gz: 0c8f579f4b9eca0402d1a28c986ad188d72b2dac57a8f4cc92d18a17ea235d7fe7ebd03a3f38dcd17e91496adc595ce347b8a1c73f449d48008e54655c5ce9e1

data/README.adoc

@@ -89,10 +89,12 @@ end
 
 [source,ruby]
 ----
-hit_collection = RelatonNist::NistBibliography.search("8200")
-=> <RelatonNist::HitCollection:0x007fc069d19da0 @ref=8200 @fetched=false>
+hit_collection = RelatonNist::NistBibliography.search("NISTIR 8200")
+[relaton-nist] (NIST IR 8200) Fetching from csrc.nist.gov ...
+[relaton-nist] (NIST IR 8200) Fetching from Relaton repository ...
+=> <RelatonNist::HitCollection:0x00000000004b28 @ref=NIST IR 8200 @fetched=false>
 
-item = hit_collection[1].fetch
+item = hit_collection[0].fetch
 => #<RelatonNist::NistBibliographicItem:0x007fc049aa6778
  ...
 ----
@@ -101,9 +103,9 @@ item = hit_collection[1].fetch
 [source,ruby]
 ----
 item.to_xml
-=> "<bibitem id="SP800-160Vol.1Ver.1Rev.1" type="standard" schema-version="v1.2.1">
-      <fetched>2022-12-05</fetched>
-      <title format="text/plain" language="en" script="Latn">Engineering Trustworthy Secure Systems</title>
+=> "<bibitem id="NISTIR8200" type="standard" schema-version="v1.2.4">
+      <fetched>2023-10-16</fetched>
+      <title format="text/plain" language="en" script="Latn">Interagency report on the status of international cybersecurity standardization for the internet of things (IoT)</title>
       ...
     <bibitem>"
 ----
@@ -111,9 +113,9 @@ With argument `bibdata: true` it outputs XML wrapped by `bibdata` element and ad
 [source,ruby]
 ----
 item.to_xml bibdata: true
-=> "<bibdata type="standard" schema-version="v1.2.1">
-      <fetched>2022-12-05</fetched>
-      <title format="text/plain" language="en" script="Latn">Engineering Trustworthy Secure Systems</title>
+=> "<bibdata type="standard" schema-version="v1.2.4">
+      <fetched>2023-10-16</fetched>
+      <title format="text/plain" language="en" script="Latn">Interagency report on the status of international cybersecurity standardization for the internet of things (IoT)</title>
       ...
       <ext schema-version="v1.0.0">
         <doctype>standard</doctype>
@@ -124,9 +126,10 @@ item.to_xml bibdata: true
 === Get code, and year
 [source,ruby]
 ----
-RelatonNist::NistBibliography.get("NIST IR 8200", "2018", {})
-[relaton-nist] (NIST IR 8200) fetching...
-[relaton-nist] (NIST IR 8200) found `NIST IR 8200`
+RelatonNist::NistBibliography.get("NIST IR 8200", "2018")
+[relaton-nist] (NIST SP 8200:2018) Fetching from csrc.nist.gov ...
+[relaton-nist] (NIST IR 8200:2018) Fetching from Relaton repository...
+[relaton-nist] (NIST IR 8200:2018) Found: `NIST IR 8200`
 => #<RelatonNist::NistBibliographicItem:0x00007fab74a572c0
 ...
 ----
@@ -153,10 +156,10 @@ NIST FIPS {docnumber}
 ----
 [source,ruby]
 ----
-RelatonNist::NistBibliography.get("SP 800-205 (February 2019) (PD)")
-[relaton-nist] (SP 800-205) fetching...
-[relaton-nist] (SP 800-205) found `SP 800-205 (Draft)`
-=> #<RelatonNist::NistBibliographicItem:0x007fc059934768
+RelatonNist::NistBibliography.get("NIST SP 800-205 (February 2019) (IPD)")
+[relaton-nist] (NIST SP 800-205) Fetching from csrc.nist.gov ...
+[relaton-nist] (NIST SP 800-205) Found: `NIST SP 800-205 (Draft)`
+=> #<RelatonNist::NistBibliographicItem:0x00000001105afdc8
 ...
 ----
 
@@ -172,22 +175,22 @@ Referehces can contain optional parameters `{ptN}{vN}{verN}{rN}{/Add}`:
 [source,ruby]
 ----
 item = RelatonNist::NistBibliography.get 'NIST SP 800-67r1'
-[relaton-nist] (NIST SP 800-67r1) fetching...
-[relaton-nist] (NIST SP 800-67r1) found `SP 800-67 Rev. 1`
-=> #<RelatonNist::NistBibliographicItem:0x00007fab748ae978
+[relaton-nist] (NIST SP 800-67r1) Fetching from csrc.nist.gov ...
+[relaton-nist] (NIST SP 800-67r1) Found: `NIST SP 800-67 Rev. 1`
+=> #<RelatonNist::NistBibliographicItem:0x00000001105acd08
 ...
 
 item.docidentifier.first.id
-=> "SP 800-67 Rev. 1"
+=> "NIST SP 800-67 Rev. 1"
 
-item = RelatonNist::NistBibliography.get 'SP 800-38A/Add'
-[relaton-nist] (SP 800-38A/Add) fetching...
-[relaton-nist] (SP 800-38A/Add) found `SP 800-38A-Add`
-=> #<RelatonNist::NistBibliographicItem:0x007fd88c21d880
+item = RelatonNist::NistBibliography.get 'NIST SP 800-38A/Add'
+[relaton-nist] (NIST SP 800-38A/Add) Fetching from csrc.nist.gov ...
+[relaton-nist] (NIST SP 800-38A/Add) Found: `NIST SP 800-38A-Add`
+=> #<RelatonNist::NistBibliographicItem:0x00000001105abf48
 ...
 
 item.docidentifier.first.id
-=> "SP 800-38A-Add"
+=> "NIST SP 800-38A-Add"
 ----
 
 === Typed links
@@ -197,8 +200,12 @@ NIST documents may have `src` and `doi` link types.
 [source,ruby]
 ----
 item.link
-=> [#<RelatonBib::TypedUri:0x00007f901971dc10 @content=#<Addressable::URI:0x62c URI:https://csrc.nist.gov/publications/detail/sp/800-67/rev-2/final>, @type="src">,
- #<RelatonBib::TypedUri:0x00007f901971d6e8 @content=#<Addressable::URI:0x640 URI:https://doi.org/10.6028/NIST.SP.800-67r2>, @type="doi">]
+=> [#<RelatonBib::TypedUri:0x0000000111087a98
+  @content=#<Addressable::URI:0x8c0 URI:https://csrc.nist.gov/pubs/sp/800/38/a/sup/final>,
+  @language=nil,
+  @script=nil,
+  @type="src">,
+ #<RelatonBib::TypedUri:0x00000001110879a8 @content=#<Addressable::URI:0x8d4 URI:https://doi.org/10.6028/NIST.SP.800-38A-Add>, @language=nil, @script=nil, @type="doi">]
 ----
 
 === Create bibliographic item from YAML

data/lib/relaton_nist/hit_collection.rb

@@ -12,6 +12,9 @@ module RelatonNist
     GHNISTDATA = "https://raw.githubusercontent.com/relaton/relaton-data-nist/main/"
     INDEX_FILE = "index-v1.yaml"
 
+    attr_reader :reference
+    attr_accessor :array
+
     #
     # @param [String] text reference
     # @param [String, nil] year reference
@@ -20,6 +23,7 @@ module RelatonNist
     #
     def initialize(text, year = nil, opts = {})
       super text, year
+      @reference = year && !text.match?(/:\d{4}$/) ? "#{text}:#{year}" : text
       @opts = opts
     end
 
@@ -54,16 +58,21 @@ module RelatonNist
     # @return [Array<RelatonNist::Hit>] hits
     #
     def search_filter # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity,Metrics/MethodLength
-      @array.select do |item|
+      arr = @array.select do |item|
         parts = doi_parts(item.hit[:json]) || code_parts(item.hit[:code])
+        iteration = item.hit.dig(:json, "iteration")
+        draft = parts[:draft] || (!iteration.nil? && iteration.match?(/\wpd/i))
         refparts[:code] && [parts[:series], item.hit[:series]].include?(refparts[:series]) &&
           refparts[:code].casecmp(parts[:code].upcase).zero? &&
           refparts[:prt] == parts[:prt] &&
           (refparts[:vol].nil? || refparts[:vol] == parts[:vol]) &&
           (refparts[:ver].nil? || refparts[:ver] == parts[:ver]) &&
           (refparts[:rev].nil? || refparts[:rev] == parts[:rev]) &&
-          refparts[:draft] == parts[:draft] && refparts[:add] == parts[:add]
+          refparts[:draft] == draft && refparts[:add] == parts[:add]
       end
+      dup = self.dup
+      dup.array = arr
+      dup
     end
 
     private
@@ -99,7 +108,7 @@ module RelatonNist
         ver: match(/v(?:er)?(?<val>[\d.]+)/, id),
         rev: match(/r(?<val>\d+)/, id),
         add: match(/-Add(?<val>\d*)/, id),
-        draft: !match(/\.ipd/, id).nil?,
+        draft: !match(/\.ipd|-draft/, id).nil?,
       }
     end
 
@@ -181,7 +190,10 @@ module RelatonNist
     # @raise [OpenURI::HTTPError] if GitHub repo is not available
     #
     def from_ga # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      Util.warn "(#{@reference}) Fetching from Relaton repository ..."
       ref = full_ref
+      return [] if ref.empty?
+
       # fn = ref.gsub(%r{[/\s:.]}, "_").upcase
       index = Relaton::Index.find_or_create :nist, url: "#{GHNISTDATA}index-v1.zip", file: INDEX_FILE
       rows = index.search(ref).sort_by { |r| r[:id] }
@@ -209,20 +221,20 @@ module RelatonNist
     #
     # @return [Array<RelatonNist::Hit>] hits
     #
-    def from_json
+    def from_json # rubocop:disable Metrics/AbcSize
+      Util.warn "(#{@reference}) Fetching from csrc.nist.gov ..."
       select_data.map do |h|
         /(?<series>(?<=-)\w+$)/ =~ h["series"]
         title = [h["title-main"], h["title-sub"]].compact.join " - "
         release_date = RelatonBib.parse_date h["published-date"], false
-        Hit.new({ code: docidentifier(h), series: series.upcase, title: title,
-                  url: h["uri"], status: h["status"],
-                  release_date: release_date, json: h }, self)
+        Hit.new({ code: docidentifier(h), series: series.upcase, title: title, url: h["uri"],
+                  status: h["status"], release_date: release_date, json: h }, self)
       end
     end
 
     def docidentifier(json) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
-      parts = doi_parts json
-      return json["docidentifier"] unless parts
+      parts = doi_parts(json) || code_parts(json["docidentifier"])
+      # return json["docidentifier"] unless parts
 
       id = parts[:code]
       id = "NIST #{parts[:series]} #{id}" if parts[:series]

data/lib/relaton_nist/nist_bibliography.rb

@@ -84,10 +84,10 @@ module RelatonNist
         result = nistbib_search_filter(code, year, opts) || (return nil)
         ret = nistbib_results_filter(result, year, opts)
         if ret[:ret]
-          Util.warn "(#{code}) found `#{ret[:ret].docidentifier.first.id}`"
+          Util.warn "(#{result.reference}) Found: `#{ret[:ret].docidentifier.first.id}`"
           ret[:ret]
         else
-          fetch_ref_err(code, year, ret[:years])
+          fetch_ref_err(result.reference, year, ret[:years])
         end
       end
 
@@ -114,20 +114,19 @@ module RelatonNist
                     when "F" then "final"
                     else iter
                     end
-        result.each_slice(3) do |s| # ISO website only allows 3 connections
-          fetch_pages(s, 3).each_with_index do |r, _i|
-            if opts[:date]
-              dates = r.date.select { |d| d.on(:date) == opts[:date] }
-              next if dates.empty?
-            end
-            next if iter && r.status.iteration != iteration
-            return { ret: r } if !year
+        result.each do |h|
+          r = h.fetch
+          if opts[:date]
+            dates = r.date.select { |d| d.on(:date) == opts[:date] }
+            next if dates.empty?
+          end
+          next if iter && r.status.iteration != iteration
+          return { ret: r } if !year
 
-            r.date.select { |d| d.type == "published" }.each do |d|
-              return { ret: r } if year.to_i == d.on(:year)
+          r.date.select { |d| d.type == "published" }.each do |d|
+            return { ret: r } if year.to_i == d.on(:year)
 
-              missed_years << d.on(:year)
-            end
+            missed_years << d.on(:year)
           end
         end
         { years: missed_years }
@@ -141,13 +140,13 @@ module RelatonNist
       #
       # @return [Array<RelatonNist::NistBibliographicItem>] bibliographic items
       #
-      def fetch_pages(hits, threads)
-        workers = RelatonBib::WorkersPool.new threads
-        workers.worker { |w| { i: w[:i], hit: w[:hit].fetch } }
-        hits.each_with_index { |hit, i| workers << { i: i, hit: hit } }
-        workers.end
-        workers.result.sort_by { |a| a[:i] }.map { |x| x[:hit] }
-      end
+      # def fetch_pages(hits, threads)
+      #   workers = RelatonBib::WorkersPool.new threads
+      #   workers.worker { |w| { i: w[:i], hit: w[:hit].fetch } }
+      #   hits.each_with_index { |hit, i| workers << { i: i, hit: hit } }
+      #   workers.end
+      #   workers.result.sort_by { |a| a[:i] }.map { |x| x[:hit] }
+      # end
 
       #
       # Get search results and filter them by code and year
@@ -160,7 +159,6 @@ module RelatonNist
       # @return [RelatonNist::HitCollection] hits collection
       #
       def nistbib_search_filter(code, year, opts)
-        Util.warn "(#{code}) fetching..."
         result = search(code, year, opts)
         result.search_filter
       end
@@ -168,21 +166,19 @@ module RelatonNist
       #
       # Outputs warning message if no match found
       #
-      # @param [String] code reference
+      # @param [String] ref reference
       # @param [String, nil] year year
       # @param [Array<String>] missed_years missed years
       #
       # @return [nil] nil
       #
-      def fetch_ref_err(code, year, missed_years) # rubocop:disable Metrics/MethodLength
-        id = year ? "#{code}:#{year}" : code
-        Util.warn "WARNING: no match found online for `#{id}`. " \
-                  "The code must be exactly like it is on the standards website."
+      def fetch_ref_err(ref, year, missed_years)
+        Util.warn "(#{ref}) No found."
         unless missed_years.empty?
           Util.warn "(There was no match for #{year}, though there " \
                     "were matches found for `#{missed_years.join('`, `')}`.)"
         end
-        if /\d-\d/.match? code
+        if /\d-\d/.match? ref
           Util.warn "The provided document part may not exist, " \
                     "or the document may no longer be published in parts."
         end

data/lib/relaton_nist/scrapper.rb

@@ -51,7 +51,7 @@ module RelatonNist
           script: [json["script"]],
           docstatus: fetch_status(json), # hit_data[:status]),
           copyright: fetch_copyright(json["published-date"]),
-          relation: fetch_relations_json(json),
+          relation: fetch_relations_json(hit_data),
           place: ["Gaithersburg, MD"],
           keyword: fetch_keywords(json),
           commentperiod: fetch_commentperiod_json(json),
@@ -80,10 +80,18 @@ module RelatonNist
       def fetch_status(doc)
         stage = doc["status"]
         subst = doc["substage"]
-        iter = doc["iteration"] == "initial" ? 1 : doc["iteration"]
+        iter = iteration(doc["iteration"])
         RelatonNist::DocumentStatus.new stage: stage, substage: subst, iteration: iter.to_s
       end
 
+      def iteration(iter)
+        case iter
+        when "initial", "ipd" then 1
+        when /(\d)pd/i then $1
+        else iter
+        end
+      end
+
       # Fetch titles.
       # @param hit_data [Hash]
       # @return [Array<Hash>]
@@ -109,7 +117,7 @@ module RelatonNist
         #   d = doc.at("//span[@id='pub-release-date']")&.text&.strip
         #   issued = RelatonBib.parse_date d
         # end
-        dates << { type: "issued", on: issued.to_s }
+        dates << { type: "issued", on: issued.to_s } if issued
         dates
       end
 
@@ -182,9 +190,9 @@ module RelatonNist
       # @return [String, NilClass]
       def fetch_edition(doc)
         # if doc.is_a? Hash
-        return unless doc["edition"]
+        return unless doc["edition"] || doc["revision"]
 
-        rev = doc["edition"]
+        rev = doc["edition"] || doc["revision"]
         "Revision #{rev}"
       end
 
@@ -210,13 +218,14 @@ module RelatonNist
         links
       end
 
-      def fetch_relations_json(doc)
+      def fetch_relations_json(hit)
+        doc = hit[:json]
         relations = doc["supersedes"].map do |r|
-          doc_relation "supersedes", r["docidentifier"], r["uri"]
+          doc_relation "supersedes", hit[:code], r["uri"]
         end
 
         relations + doc["superseded-by"].map do |r|
-          doc_relation "updates", r["docidentifier"], r["uri"]
+          doc_relation "updates", hit[:code], r["uri"]
         end
       end
 
@@ -230,16 +239,11 @@ module RelatonNist
           t = "obsoletes"
         else t = type
         end
-        DocumentRelation.new(
-          type: t,
-          description: descr,
-          bibitem: RelatonBib::BibliographicItem.new(
-            formattedref: RelatonBib::FormattedRef.new(
-              content: ref, language: lang, script: script, format: "text/plain",
-            ),
-            link: [RelatonBib::TypedUri.new(type: "src", content: uri)],
-          ),
-        )
+        ids = [RelatonBib::DocumentIdentifier.new(id: ref, type: "NIST", primary: true)]
+        fref = RelatonBib::FormattedRef.new(content: ref, language: lang, script: script, format: "text/plain")
+        link = [RelatonBib::TypedUri.new(type: "src", content: uri)]
+        bib = RelatonBib::BibliographicItem.new(formattedref: fref, link: link, docid: ids)
+        DocumentRelation.new(type: t, description: descr, bibitem: bib)
       end
 
       # @param doc [Hash]

data/lib/relaton_nist/version.rb

@@ -1,3 +1,3 @@
 module RelatonNist
-  VERSION = "1.16.3".freeze
+  VERSION = "1.16.4".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-nist
 version: !ruby/object:Gem::Version
-  version: 1.16.3
+  version: 1.16.4
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-10-04 00:00:00.000000000 Z
+date: 2023-10-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: relaton-bib