relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.1

data/lib/relaton/nist/bibliography.rb

@@ -0,0 +1,191 @@
+module Relaton
+  module Nist
+    class Bibliography
+      extend Core::DateParser
+
+      class << self
+        #
+        # Search NIST documents by reference
+        #
+        # @param text [String] reference
+        #
+        # @return [Relaton::Nist::HitCollection] search result
+        #
+        def search(text, year = nil, opts = {})
+          ref = text.sub(/^NISTIR/, "NIST IR").sub(/\/Add/, " Add")
+          # pubid 2.x only recognizes the addendum marker with a trailing
+          # period, and only splits an uppercase part letter — canonicalize
+          # both ("800-38a Add" -> "800-38A Add.") so @reference parses to the
+          # same pubid the index/CSRC carry.
+          ref = ref.sub(/\bAdd\b\.?/i, "Add.").sub(/([0-9])([a-z])(?=\s+Add\.)/) { "#{$1}#{$2.upcase}" }
+          HitCollection.search ref, year, opts
+        rescue OpenURI::HTTPError, SocketError, OpenSSL::SSL::SSLError => e
+          raise Relaton::RequestError, e.message
+        end
+
+        #
+        # Get NIST document by reference
+        #
+        # @param code [String] the NIST standard Code to look up
+        # @param year [String] the year the standard was published (optional)
+        # @param opts [Hash] options
+        # @option opts [Boolean] :all_parts restricted to all parts
+        #
+        # @return [Relaton::Nist::ItemData, nil] bibliographic item
+        #
+        def get(code, year = nil, opts = {}) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+          return fetch_ref_err(code, year, []) if code.match?(/\sEP$/)
+
+          /^(?<code2>[^(]+)(?:\((?<date2>\w+\s(?:\d{2},\s)?\d{4})\))?\s?\(?(?:(?<=\()(?<stage>(?:I|F|\d)PD))?/ =~ code
+          stage ||= /(?<=\.)PD-\w+(?=\.)/.match(code)&.to_s
+          if code2
+            code = code2.strip
+            opts[:date] = parse_date(date2, str: false) if date2
+            opts[:stage] = stage if stage
+          end
+
+          if year.nil?
+            /^(?<code1>[^:]+):(?<year1>[^:]+)$/ =~ code
+            unless code1.nil?
+              code = code1
+              year = year1
+            end
+          end
+
+          code += "-1" if opts[:all_parts]
+          nistbib_get(code, year, opts)
+        end
+
+        private
+
+        #
+        # Get NIST document by reference
+        #
+        # @param [String] code reference
+        # @param [String] year year
+        # @param [Hash] opts options
+        #
+        # @return [Relaton::Nist::ItemData, nil] bibliographic item
+        #
+        def nistbib_get(code, year, opts)
+          result = nistbib_search_filter(code, year, opts) || (return nil)
+          ret = nistbib_results_filter(result, year, opts)
+          if ret[:ret]
+            Util.info "Found: `#{ret[:ret].docidentifier.first.content}`", key: result.reference
+            ret[:ret]
+          else
+            fetch_ref_err(result.reference, year, ret[:years])
+          end
+        end
+
+        #
+        # Sort through results, return first match by code, year, and title
+        #
+        # @param opts [Hash] options
+        # @option opts [Date] :date date filter
+        # @option opts [String] :stage stage filter
+        #
+        # @return [Hash] result
+        #
+        def nistbib_results_filter(result, year, opts)
+          missed_years = []
+          iteration = parse_iteration(opts[:stage])
+
+          # A stageless query means "the published document". Multiple
+          # editions can match once stage is excluded (e.g. a final "r2"
+          # and its draft "Rev. 2 ipd"), so try published hits before
+          # drafts. An explicit stage keeps the original order. Stable
+          # partition preserves existing tie-order among finals.
+          ordered = if iteration
+                      result.array
+                    else
+                      result.array.sort_by.with_index { |h, i| [draft_hit?(h) ? 1 : 0, i] }
+                    end
+
+          ordered.each do |h|
+            r = h.item
+            next if opts[:date] && !match_date?(r, opts[:date])
+            next if iteration && r.status&.iteration != iteration
+            return { ret: r } unless year
+            next unless match_year?(r, year) { |y| missed_years << y }
+
+            return { ret: r }
+          end
+          { years: missed_years }
+        end
+
+        # A hit is a draft when its CSRC status says so (e.g. "draft-public")
+        # or its rendered code carries a PD stage marker (ipd/fpd/NpD).
+        def draft_hit?(hit)
+          hit.hit[:status].to_s.include?("draft") ||
+            hit.hit[:code].to_s.match?(/(?:\bi|\bf|\b\d)pd\b|\(Draft\)/i)
+        end
+
+        def parse_iteration(stage)
+          iter = /\w+(?=PD)|(?<=PD-)\w+/.match(stage)&.to_s
+          case iter
+          when "I" then "1"
+          when "F" then "final"
+          else iter
+          end
+        end
+
+        def match_date?(item, date)
+          item.date.any? do |d|
+            date_val = d.at || d.from
+            date_val && parse_date(date_val.to_s, str: false) == date
+          end
+        end
+
+        def match_year?(item, year)
+          item.date.select { |d| d.type == "published" || d.type == "issued" }.each do |d|
+            date_val = d.at || d.from
+            next unless date_val
+
+            parsed_year = parse_date(date_val.to_s, str: false)&.year
+            return parsed_year if year.to_i == parsed_year
+
+            yield parsed_year if block_given?
+          end
+          nil
+        end
+
+        #
+        # Get search results and filter them by code and year
+        #
+        # @param code [String] reference
+        # @param year [String, nil] year
+        # @param opts [Hash] options
+        #
+        # @return [Relaton::Nist::HitCollection] hits collection
+        #
+        def nistbib_search_filter(code, year, opts)
+          result = search(code, year, opts)
+          result.search_filter
+        end
+
+        #
+        # Outputs warning message if no match found
+        #
+        # @param [String] ref reference
+        # @param [String, nil] year year
+        # @param [Array<String>] missed_years missed years
+        #
+        # @return [nil] nil
+        #
+        def fetch_ref_err(ref, year, missed_years)
+          Util.info "Not found.", key: ref
+          unless missed_years.empty?
+            Util.info "(There was no match for #{year}, though there " \
+                      "were matches found for `#{missed_years.join('`, `')}`.)", key: ref
+          end
+          if /\d-\d/.match? ref
+            Util.info "The provided document part may not exist, " \
+                      "or the document may no longer be published in parts.", key: ref
+          end
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/relaton/nist/comment_period.rb

@@ -0,0 +1,15 @@
+module Relaton
+  module Nist
+    class CommentPeriod < Lutaml::Model::Serializable
+      attribute :from, :date
+      attribute :to, :date
+      attribute :extended, :date
+
+      xml do
+        map_element "from", to: :from
+        map_element "to", to: :to
+        map_element "extended", to: :extended
+      end
+    end
+  end
+end

data/lib/relaton/nist/data_fetcher.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require "yaml"
+require "mechanize"
+require "loc_mods"
+require_relative "../nist"
+require_relative "mods_parser"
+
+module Relaton
+  module Nist
+    class DataFetcher < Core::DataFetcher
+      URL = "https://github.com/usnistgov/NIST-Tech-Pubs/releases/download/Nov2024/allrecords-MODS.xml"
+
+      def fetch(_source = nil)
+        FileUtils.rm Dir[File.join(@output, "*.#{@ext}")]
+        fetch_tech_pubs
+        # add_static_files
+        index.save
+        report_errors
+      end
+
+      def fetch_tech_pubs
+        xml_data = Mechanize.new.get(URL).body
+        docs = LocMods::Collection.from_xml xml_data
+        docs.mods.each { |doc| write_file ModsParser.new(doc, series, @errors).parse }
+      end
+
+      def write_file(bib)
+        id = bib.docidentifier.find(&:primary) || bib.docidentifier.first
+        file = output_file id.content.sub(/^NIST IR/, "NISTIR")
+        if @files.include? file
+          Util.warn "File #{file} exists. Docid: #{id.content}"
+        else @files << file
+        end
+        pid = pubid id.content
+        index.add_or_update pid, file if pid
+        File.write file, serialize(bib), encoding: "UTF-8"
+      end
+
+      # Parse a docidentifier string into a Pubid::Nist::Identifier; nil (with a
+      # warning) if pubid can't parse it, so a single bad id never aborts the
+      # crawl or corrupts index-v2.
+      def pubid(id)
+        ::Pubid::Nist::Identifier.parse id
+      rescue StandardError => e
+        Util.warn "Failed to parse `#{id}` with pubid: #{e.message}"
+        nil
+      end
+
+      # def add_static_files
+      #   Dir["./static/*.yaml"].each do |file|
+      #     bib = Item.from_yaml(File.read(file, encoding: "UTF-8"))
+      #     index.add_or_update bib.docidentifier[0].content, file
+      #   end
+      # end
+
+      def to_yaml(bib)
+        Item.to_yaml(bib)
+      end
+
+      def to_xml(bib)
+        Bibdata.to_xml(bib)
+      end
+
+      def to_bibxml(bib)
+        bib.to_rfcxml
+      end
+
+      def log_error(msg)
+        Util.error msg
+      end
+
+      def index
+        @index ||= Relaton::Index.find_or_create(
+          :nist, file: "#{INDEXFILE}.yaml", pubid_class: ::Pubid::Nist::Identifier
+        )
+      end
+
+      def series
+        @series ||= YAML.load_file File.expand_path("series.yaml", __dir__)
+      end
+    end
+  end
+end

data/lib/relaton/nist/date.rb

@@ -0,0 +1,7 @@
+module Relaton
+  module Nist
+    class Date < Bib::Date
+      attribute :type, :string, values: %w[abandoned superseded]
+    end
+  end
+end

data/lib/relaton/nist/doctype.rb

@@ -0,0 +1,7 @@
+module Relaton
+  module Nist
+    class Doctype < Bib::Doctype
+      attribute :content, :string, values: %w[standard]
+    end
+  end
+end

data/lib/relaton/nist/ext.rb

@@ -0,0 +1,16 @@
+require_relative "doctype"
+require_relative "comment_period"
+
+module Relaton
+  module Nist
+    class Ext < Bib::Ext
+      attribute :doctype, Doctype
+      attribute :commentperiod, CommentPeriod
+
+      xml { map_element "commentperiod", to: :commentperiod }
+      key_value { map_element "commentperiod", to: :commentperiod }
+
+      def get_schema_version = Relaton.schema_versions["relaton-model-nist"]
+    end
+  end
+end

data/lib/relaton/nist/hit.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Nist
+    class Hit < Core::Hit
+      attr_writer :item
+
+      #
+      # Parse page.
+      #
+      # @return [Relaton::Nist::ItemData] bibliographic item
+      #
+      def item
+        @item ||= Scraper.parse_page @hit
+      end
+    end
+  end
+end