relaton-nist 1.8.0 → 1.9.3

data/lib/relaton_nist/hit_collection.rb

@@ -13,75 +13,90 @@ module RelatonNist
     PUBS_EXPORT = URI.join(DOMAIN, "/CSRC/media/feeds/metanorma/pubs-export")
     DATAFILEDIR = File.expand_path ".relaton/nist", Dir.home
     DATAFILE = File.expand_path "pubs-export.zip", DATAFILEDIR
+    GHNISTDATA = "https://raw.githubusercontent.com/relaton/relaton-data-nist/main/data/"
 
-    # @param ref_nbr [String]
-    # @param year [String]
-    # @param opts [Hash]
-    # @option opts [String] :stage
-    def initialize(ref_nbr, year = nil, opts = {}) # rubocop:disable Metrics/AbcSize
-      super ref_nbr, year
+    def self.search(text, year = nil, opts = {})
+      new(text, year).search(opts)
+    end
+
+    def search(opts)
+      @array = from_json(**opts)
+      @array = from_ga unless @array.any?
+      sort_hits!
+    end
 
-      /(?<docid>(SP|FIPS)\s[0-9-]+)/ =~ text
-      @array = docid ? from_json(docid, **opts) : from_csrc(**opts)
-      @array = from_csrc(**opts) unless @array.any?
+    private
 
+    def sort_hits!
       @array.sort! do |a, b|
-        if a.sort_value != b.sort_value
-          b.sort_value - a.sort_value
-        else
+        if a.sort_value == b.sort_value
           (b.hit[:release_date] - a.hit[:release_date]).to_i
+        else
+          b.sort_value - a.sort_value
         end
       end
+      self
     end
 
-    private
+    def from_ga # rubocop:disable Metrics/AbcSize
+      fn = text.gsub(%r{[/\s:.]}, "_").upcase
+      yaml = OpenURI.open_uri "#{GHNISTDATA}#{fn}.yaml"
+      hash = YAML.safe_load yaml
+      bib = RelatonNist::NistBibliographicItem.from_hash hash
+      hit = Hit.new({ code: text }, self)
+      hit.fetch = bib
+      [hit]
+    rescue OpenURI::HTTPError => e
+      return [] if e.io.status[0] == "404"
+
+      raise e
+    end
 
     # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
 
     # @param stage [String]
     # @return [Array<RelatonNist::Hit>]
-    def from_csrc(**opts)
-      from, to = nil
-      if year
-        d    = Date.strptime year, "%Y"
-        from = d.strftime "%m/%d/%Y"
-        to   = d.next_year.prev_day.strftime "%m/%d/%Y"
-      end
-      url = "#{DOMAIN}/publications/search?keywords-lg=#{text}"\
-        "&sortBy-lg=relevence"
-      url += "&dateFrom-lg=#{from}" if from
-      url += "&dateTo-lg=#{to}" if to
-      url += if /PD/.match? opts[:stage]
-               "&status-lg=Draft,Retired Draft,Withdrawn"
-             else
-               "&status-lg=Final,Withdrawn"
-             end
-
-      doc = Nokogiri::HTML OpenURI.open_uri(::Addressable::URI.parse(url).normalize)
-      doc.css("table.publications-table > tbody > tr").map do |h|
-        link  = h.at("td/div/strong/a")
-        serie = h.at("td[1]").text.strip
-        code  = h.at("td[2]").text.strip
-        title = link.text
-        doc_url = DOMAIN + link[:href]
-        status = h.at("td[4]").text.strip.downcase
-        release_date = Date.strptime h.at("td[5]").text.strip, "%m/%d/%Y"
-        Hit.new(
-          {
-            code: code, serie: serie, title: title, url: doc_url,
-            status: status, release_date: release_date
-          }, self
-        )
-      end
-    end
+    # def from_csrc(**opts)
+    #   from, to = nil
+    #   if year
+    #     d    = Date.strptime year, "%Y"
+    #     from = d.strftime "%m/%d/%Y"
+    #     to   = d.next_year.prev_day.strftime "%m/%d/%Y"
+    #   end
+    #   url = "#{DOMAIN}/publications/search?keywords-lg=#{text}"\
+    #         "&sortBy-lg=relevence"
+    #   url += "&dateFrom-lg=#{from}" if from
+    #   url += "&dateTo-lg=#{to}" if to
+    #   url += if /PD/.match? opts[:stage]
+    #            "&status-lg=Draft,Retired Draft,Withdrawn"
+    #          else
+    #            "&status-lg=Final,Withdrawn"
+    #          end
+
+    #   doc = Nokogiri::HTML OpenURI.open_uri(::Addressable::URI.parse(url).normalize)
+    #   doc.css("table.publications-table > tbody > tr").map do |h|
+    #     link  = h.at("td/div/strong/a")
+    #     serie = h.at("td[1]").text.strip
+    #     code  = h.at("td[2]").text.strip
+    #     title = link.text
+    #     doc_url = DOMAIN + link[:href]
+    #     status = h.at("td[4]").text.strip.downcase
+    #     release_date = Date.strptime h.at("td[5]").text.strip, "%m/%d/%Y"
+    #     Hit.new(
+    #       {
+    #         code: code, serie: serie, title: title, url: doc_url,
+    #         status: status, release_date: release_date
+    #       }, self
+    #     )
+    #   end
+    # end
     # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
     # Fetches data form json
-    # @param docid [String]
     # @param stage [String]
     # @return [Array<RelatonNist::Hit>]
-    def from_json(docid, **opts)
-      select_data(docid, **opts).map do |h|
+    def from_json(**opts)
+      select_data(**opts).map do |h|
         /(?<serie>(?<=-)\w+$)/ =~ h["series"]
         title = [h["title-main"], h["title-sub"]].compact.join " - "
         release_date = RelatonBib.parse_date h["published-date"], false
@@ -91,22 +106,20 @@ module RelatonNist
       end
     end
 
-    # @param docid [String]
     # @param stage [String]
     # @return [Array<Hach>]
-    def select_data(docid, **opts) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength,Metrics/PerceivedComplexity
-      # ref = docid.sub(/(?<=\d{3}-\d{2})r(\d+)/, ' Rev. \1')
+    def select_data(**opts) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength,Metrics/PerceivedComplexity
       d = Date.strptime year, "%Y" if year
-      # didrx = Regexp.new(docid)
+      statuses = %w[draft-public draft-prelim]
       data.select do |doc|
         next unless match_year?(doc, d)
 
         if /PD/.match? opts[:stage]
-          next unless %w[draft-public draft-prelim].include? doc["status"]
+          next unless statuses.include? doc["status"]
         else
           next unless doc["status"] == "final"
         end
-        doc["docidentifier"].include? docid
+        doc["docidentifier"].include? text
       end
     end
 
@@ -134,8 +147,8 @@ module RelatonNist
     #
     # @prarm ctime [Time, NilClass]
     def fetch_data(ctime)
-      resp = OpenURI.open_uri("#{PUBS_EXPORT}.meta")
-      if !ctime || ctime < resp.last_modified
+      # resp = OpenURI.open_uri("#{PUBS_EXPORT}.meta")
+      if !ctime || ctime < OpenURI.open_uri("#{PUBS_EXPORT}.meta").last_modified
         @data = nil
         uri_open = URI.method(:open) || Kernel.method(:open)
         FileUtils.mkdir_p DATAFILEDIR unless Dir.exist? DATAFILEDIR

data/lib/relaton_nist/nist_bibliographic_item.rb

@@ -67,7 +67,7 @@ module RelatonNist
     # @return [RelatonNist::GbBibliographicItem]
     def self.from_hash(hash)
       item_hash = RelatonNist::HashConverter.hash_to_bib(hash)
-      new **item_hash
+      new(**item_hash)
     end
 
     # @param opts [Hash]

data/lib/relaton_nist/nist_bibliography.rb

@@ -14,9 +14,9 @@ module RelatonNist
       # @param text [String]
       # @return [RelatonNist::HitCollection]
       def search(text, year = nil, opts = {})
-        HitCollection.new text, year, opts
-      rescue OpenURI::HTTPError, SocketError, OpenSSL::SSL::SSLError
-        raise RelatonBib::RequestError, "Could not access https://www.nist.gov"
+        HitCollection.search text, year, opts
+      rescue OpenURI::HTTPError, SocketError, OpenSSL::SSL::SSLError => e
+        raise RelatonBib::RequestError, e.message
       end
 
       # @param code [String] the NIST standard Code to look up (e..g "8200")
@@ -174,7 +174,17 @@ module RelatonNist
         }
         ref = matches[:code] ? "#{matches[:serie]} #{matches[:code]}" : code
         result = search(ref, year, opts)
-        result.select { |i| search_filter i, matches, code }
+        selected_result = result.select { |i| search_filter i, matches, code }
+        return selected_result if selected_result.any? || !matches[:code]
+
+        search full_ref(matches)
+      end
+
+      def full_ref(matches)
+        ref = "#{matches[:serie]} #{matches[:code]}"
+        ref += long_to_short(matches[:prt1], matches[:prt2]).to_s
+        ref += long_to_short(matches[:vol1], matches[:vol2]).to_s
+        ref
       end
 
       def match(regex, code)
@@ -192,11 +202,11 @@ module RelatonNist
           (?<code>[0-9-]{3,}[A-Z]?)
           (?<prt1>pt\d+)?
           (?<vol1>v\d+)?
-          (?<ver1>ver[\d\.]+)?
+          (?<ver1>ver[\d.]+)?
           (?<rev1>r\d+)?
           (\s(?<prt2>Part\s\d+))?
           (\s(?<vol2>Vol\.\s\d+))?
-          (\s(?<ver2>(Ver\.|Version)\s[\d\.]+))?
+          (\s(?<ver2>(Ver\.|Version)\s[\d.]+))?
           (\s(?<rev2>Rev\.\s\d+))?
           (\s(?<add>Add)endum)?
         }x =~ item.hit[:code]

data/lib/relaton_nist/processor.rb

@@ -2,11 +2,12 @@ require "relaton/processor"
 
 module RelatonNist
   class Processor < Relaton::Processor
-    def initialize
+    def initialize # rubocop:disable Lint/MissingSuper
       @short = :relaton_nist
       @prefix = "NIST"
       @defaultprefix = %r{^(NIST|NISTGCR|ITL Bulletin|JPCRD|NISTIR|CSRC|FIPS)(/[^\s])?\s}
       @idtype = "NIST"
+      @datasets = %w[nist-tech-pubs]
     end
 
     # @param code [String]
@@ -17,6 +18,18 @@ module RelatonNist
       ::RelatonNist::NistBibliography.get(code, date, opts)
     end
 
+    #
+    # Fetch all the documents from a source
+    #
+    # @param [String] _source source name
+    # @param [Hash] opts
+    # @option opts [String] :output directory to output documents
+    # @option opts [String] :format
+    #
+    def fetch_data(_source, opts)
+      DataFetcher.fetch(**opts)
+    end
+
     # @param xml [String]
     # @return [RelatonNist::GbBibliographicItem]
     def from_xml(xml)