relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/logger/pool.rb

@@ -0,0 +1,24 @@
+module Relaton::Logger
+  class Pool
+    extend Forwardable
+    def_delegators :@loggers, :[], :[]=, :size, :each
+
+    attr_accessor :loggers
+
+    def initialize
+      @loggers = {}
+    end
+
+    def unknown(message = nil, progname = nil, **args, &block)
+      @loggers.each { |_, logger| logger.send(__callee__, message, progname, **args, &block) }
+      nil
+    end
+
+    %i[fatal error warn info debug].each { |m| alias_method m, :unknown }
+
+    def truncate
+      @loggers.each { |_, logger| logger.truncate }
+      nil
+    end
+  end
+end

data/lib/relaton/logger.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "forwardable"
+require "logger"
+require_relative "version"
+require_relative "logger/log"
+require_relative "logger/log_device"
+require_relative "logger/pool"
+require_relative "logger/formatter_string"
+require_relative "logger/formatter_json"
+require_relative "logger/config"
+require_relative "logger/channels/gh_issue"
+
+module Relaton
+  def self.logger_pool
+    Logger.configuration.logger_pool
+  end
+end

data/lib/relaton/nist/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Nist
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/nist/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Nist
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/nist/bibliography.rb

@@ -0,0 +1,191 @@
+module Relaton
+  module Nist
+    class Bibliography
+      extend Core::DateParser
+
+      class << self
+        #
+        # Search NIST documents by reference
+        #
+        # @param text [String] reference
+        #
+        # @return [Relaton::Nist::HitCollection] search result
+        #
+        def search(text, year = nil, opts = {})
+          ref = text.sub(/^NISTIR/, "NIST IR").sub(/\/Add/, " Add")
+          # pubid 2.x only recognizes the addendum marker with a trailing
+          # period, and only splits an uppercase part letter — canonicalize
+          # both ("800-38a Add" -> "800-38A Add.") so @reference parses to the
+          # same pubid the index/CSRC carry.
+          ref = ref.sub(/\bAdd\b\.?/i, "Add.").sub(/([0-9])([a-z])(?=\s+Add\.)/) { "#{$1}#{$2.upcase}" }
+          HitCollection.search ref, year, opts
+        rescue OpenURI::HTTPError, SocketError, OpenSSL::SSL::SSLError => e
+          raise Relaton::RequestError, e.message
+        end
+
+        #
+        # Get NIST document by reference
+        #
+        # @param code [String] the NIST standard Code to look up
+        # @param year [String] the year the standard was published (optional)
+        # @param opts [Hash] options
+        # @option opts [Boolean] :all_parts restricted to all parts
+        #
+        # @return [Relaton::Nist::ItemData, nil] bibliographic item
+        #
+        def get(code, year = nil, opts = {}) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+          return fetch_ref_err(code, year, []) if code.match?(/\sEP$/)
+
+          /^(?<code2>[^(]+)(?:\((?<date2>\w+\s(?:\d{2},\s)?\d{4})\))?\s?\(?(?:(?<=\()(?<stage>(?:I|F|\d)PD))?/ =~ code
+          stage ||= /(?<=\.)PD-\w+(?=\.)/.match(code)&.to_s
+          if code2
+            code = code2.strip
+            opts[:date] = parse_date(date2, str: false) if date2
+            opts[:stage] = stage if stage
+          end
+
+          if year.nil?
+            /^(?<code1>[^:]+):(?<year1>[^:]+)$/ =~ code
+            unless code1.nil?
+              code = code1
+              year = year1
+            end
+          end
+
+          code += "-1" if opts[:all_parts]
+          nistbib_get(code, year, opts)
+        end
+
+        private
+
+        #
+        # Get NIST document by reference
+        #
+        # @param [String] code reference
+        # @param [String] year year
+        # @param [Hash] opts options
+        #
+        # @return [Relaton::Nist::ItemData, nil] bibliographic item
+        #
+        def nistbib_get(code, year, opts)
+          result = nistbib_search_filter(code, year, opts) || (return nil)
+          ret = nistbib_results_filter(result, year, opts)
+          if ret[:ret]
+            Util.info "Found: `#{ret[:ret].docidentifier.first.content}`", key: result.reference
+            ret[:ret]
+          else
+            fetch_ref_err(result.reference, year, ret[:years])
+          end
+        end
+
+        #
+        # Sort through results, return first match by code, year, and title
+        #
+        # @param opts [Hash] options
+        # @option opts [Date] :date date filter
+        # @option opts [String] :stage stage filter
+        #
+        # @return [Hash] result
+        #
+        def nistbib_results_filter(result, year, opts)
+          missed_years = []
+          iteration = parse_iteration(opts[:stage])
+
+          # A stageless query means "the published document". Multiple
+          # editions can match once stage is excluded (e.g. a final "r2"
+          # and its draft "Rev. 2 ipd"), so try published hits before
+          # drafts. An explicit stage keeps the original order. Stable
+          # partition preserves existing tie-order among finals.
+          ordered = if iteration
+                      result.array
+                    else
+                      result.array.sort_by.with_index { |h, i| [draft_hit?(h) ? 1 : 0, i] }
+                    end
+
+          ordered.each do |h|
+            r = h.item
+            next if opts[:date] && !match_date?(r, opts[:date])
+            next if iteration && r.status&.iteration != iteration
+            return { ret: r } unless year
+            next unless match_year?(r, year) { |y| missed_years << y }
+
+            return { ret: r }
+          end
+          { years: missed_years }
+        end
+
+        # A hit is a draft when its CSRC status says so (e.g. "draft-public")
+        # or its rendered code carries a PD stage marker (ipd/fpd/NpD).
+        def draft_hit?(hit)
+          hit.hit[:status].to_s.include?("draft") ||
+            hit.hit[:code].to_s.match?(/(?:\bi|\bf|\b\d)pd\b|\(Draft\)/i)
+        end
+
+        def parse_iteration(stage)
+          iter = /\w+(?=PD)|(?<=PD-)\w+/.match(stage)&.to_s
+          case iter
+          when "I" then "1"
+          when "F" then "final"
+          else iter
+          end
+        end
+
+        def match_date?(item, date)
+          item.date.any? do |d|
+            date_val = d.at || d.from
+            date_val && parse_date(date_val.to_s, str: false) == date
+          end
+        end
+
+        def match_year?(item, year)
+          item.date.select { |d| d.type == "published" || d.type == "issued" }.each do |d|
+            date_val = d.at || d.from
+            next unless date_val
+
+            parsed_year = parse_date(date_val.to_s, str: false)&.year
+            return parsed_year if year.to_i == parsed_year
+
+            yield parsed_year if block_given?
+          end
+          nil
+        end
+
+        #
+        # Get search results and filter them by code and year
+        #
+        # @param code [String] reference
+        # @param year [String, nil] year
+        # @param opts [Hash] options
+        #
+        # @return [Relaton::Nist::HitCollection] hits collection
+        #
+        def nistbib_search_filter(code, year, opts)
+          result = search(code, year, opts)
+          result.search_filter
+        end
+
+        #
+        # Outputs warning message if no match found
+        #
+        # @param [String] ref reference
+        # @param [String, nil] year year
+        # @param [Array<String>] missed_years missed years
+        #
+        # @return [nil] nil
+        #
+        def fetch_ref_err(ref, year, missed_years)
+          Util.info "Not found.", key: ref
+          unless missed_years.empty?
+            Util.info "(There was no match for #{year}, though there " \
+                      "were matches found for `#{missed_years.join('`, `')}`.)", key: ref
+          end
+          if /\d-\d/.match? ref
+            Util.info "The provided document part may not exist, " \
+                      "or the document may no longer be published in parts.", key: ref
+          end
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/relaton/nist/comment_period.rb

@@ -0,0 +1,15 @@
+module Relaton
+  module Nist
+    class CommentPeriod < Lutaml::Model::Serializable
+      attribute :from, :date
+      attribute :to, :date
+      attribute :extended, :date
+
+      xml do
+        map_element "from", to: :from
+        map_element "to", to: :to
+        map_element "extended", to: :extended
+      end
+    end
+  end
+end

data/lib/relaton/nist/data_fetcher.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+require "yaml"
+require "mechanize"
+require "loc_mods"
+require_relative "../nist"
+require_relative "mods_parser"
+
+module Relaton
+  module Nist
+    class DataFetcher < Core::DataFetcher
+      RELEASES_URL = "https://github.com/usnistgov/NIST-Tech-Pubs/releases"
+      MODS_ASSET = "allrecords-MODS.xml"
+
+      def fetch(source = nil)
+        FileUtils.rm Dir[File.join(@output, "*.#{@ext}")]
+        fetch_tech_pubs source
+        # add_static_files
+        index.save
+        report_errors
+      end
+
+      def fetch_tech_pubs(source = nil)
+        xml_data = Mechanize.new.get(source_url(source)).body
+        docs = LocMods::Collection.from_xml xml_data
+        docs.mods.each { |doc| write_file ModsParser.new(doc, series, @errors).parse }
+      end
+
+      # Build the MODS download URL for a NIST-Tech-Pubs release. With no tag
+      # (nil, blank, or "latest") use GitHub's `latest/download` redirect so new
+      # releases are picked up automatically by the crawler; a concrete tag
+      # (e.g. "June2026") pins that specific release.
+      def source_url(source = nil)
+        tag = source.to_s.strip
+        path = tag.empty? || tag.casecmp?("latest") ? "latest/download" : "download/#{tag}"
+        "#{RELEASES_URL}/#{path}/#{MODS_ASSET}"
+      end
+
+      def write_file(bib)
+        id = bib.docidentifier.find(&:primary) || bib.docidentifier.first
+        unless id
+          failures << "Document skipped: no identifier (#{document_label(bib)})"
+          return
+        end
+        # Distinct docids can sanitize to one filename; take a path of our own
+        # rather than overwriting the other document (Core#unique_output_file).
+        docid = id.content.sub(/^NIST IR/, "NISTIR")
+        file = unique_output_file docid
+        if @files.include? file
+          # Same reserved path == same docid: a genuine duplicate. Checked FIRST,
+          # because a disambiguated path stays != output_file forever.
+          Util.warn "File #{file} exists. Docid: #{id.content}"
+        elsif file != output_file(docid)
+          Util.warn "File #{output_file docid} exists. Docid: #{id.content}. Writing #{file} instead."
+        end
+        @files << file
+        pid = pubid id.content
+        if pid
+          index.add_or_update pid, file
+        else
+          failures << "Unparseable id `#{id.content}` was not indexed (#{file})"
+        end
+        File.write file, serialize(bib), encoding: "UTF-8"
+      end
+
+      # A human-readable label for a bib with no usable docidentifier, so the
+      # skipped record is identifiable in the reported error.
+      def document_label(bib)
+        Array(bib.title).map(&:content).compact.reject(&:empty?).first || "unknown title"
+      end
+
+      # Parse a docidentifier string into a Pubid::Nist::Identifier, or nil if
+      # pubid can't parse it — so a single bad id never aborts the crawl or
+      # corrupts index-v2. The caller records the failure (see #write_file) so
+      # #report_errors can surface it as a tracked GitHub issue.
+      def pubid(id)
+        ::Pubid::Nist::Identifier.parse id
+      rescue StandardError
+        nil
+      end
+
+      # Per-document failures (unparseable ids, identifier-less records)
+      # collected during the crawl and surfaced by #report_errors.
+      def failures
+        @failures ||= []
+      end
+
+      # Surface accumulated per-document failures through the shared error
+      # machinery (the "Error fetching documents" GitHub issue in CI) so they
+      # are visible and tracked, not a hard crash or a silent stderr warning.
+      # The gh_issue channel is registered inside #report_errors, so emit these
+      # at :error (via #log_error) after it is set up and before super creates
+      # the issue.
+      def report_errors
+        gh_issue
+        failures.each { |msg| log_error msg }
+        super
+      end
+
+      # def add_static_files
+      #   Dir["./static/*.yaml"].each do |file|
+      #     bib = Item.from_yaml(File.read(file, encoding: "UTF-8"))
+      #     index.add_or_update bib.docidentifier[0].content, file
+      #   end
+      # end
+
+      def to_yaml(bib)
+        Item.to_yaml(bib)
+      end
+
+      def to_xml(bib)
+        Bibdata.to_xml(bib)
+      end
+
+      def to_bibxml(bib)
+        bib.to_rfcxml
+      end
+
+      def log_error(msg)
+        Util.error msg
+      end
+
+      def index
+        @index ||= Relaton::Index.find_or_create(
+          :nist, file: "#{INDEXFILE}.yaml", pubid_class: ::Pubid::Nist::Identifier
+        )
+      end
+
+      def series
+        @series ||= YAML.load_file File.expand_path("series.yaml", __dir__)
+      end
+    end
+  end
+end

data/lib/relaton/nist/date.rb

@@ -0,0 +1,7 @@
+module Relaton
+  module Nist
+    class Date < Bib::Date
+      attribute :type, :string, values: %w[abandoned superseded]
+    end
+  end
+end

data/lib/relaton/nist/docidentifier.rb

@@ -0,0 +1,165 @@
+module Relaton
+  module Nist
+    # NIST document identifier backed by a parsed Pubid::Nist::Identifier.
+    #
+    # Mirrors the CCSDS/IEC flavors (lib/relaton/ccsds/model/docidentifier.rb,
+    # lib/relaton/iec/model/docidentifier.rb): the lutaml `content` stays a plain
+    # string for serialization, while the parsed pubid is kept in `@pubid` so the
+    # base class's abstract structural mutators (`remove_part!`, `remove_date!`,
+    # `to_all_parts!`) can edit the identifier graph and re-render.
+    #
+    # NIST-specific caveats:
+    # - **A pubid is only adopted when it round-trips.** A NIST DOI such as
+    #   `NIST.SP.800-162` is itself a valid NIST pubid in dotted MR form, but it
+    #   must serialize verbatim (its `:human` render is `NIST SP 800-162`). So
+    #   `content=` keeps the parsed pubid only when `pubid.to_s(:human)` equals
+    #   the input; otherwise the raw string is preserved. This test is
+    #   type-independent, so it holds no matter what order lutaml assigns
+    #   `content`/`type` on `from_xml`/`from_yaml` (unlike a `type == "NIST"`
+    #   gate, which sees a nil `type` on the YAML setter-ordering path).
+    # - **The date lives in the `edition` component, not a `:date` attribute.**
+    #   NIST has no single date field (`year`/`month` scalars are nil on the
+    #   common parse paths); a publication year is carried as the edition id
+    #   (`FIPS 46e1977`) or its trailing `additional_text` (`NBS CIRC 11e2.1915`).
+    #   So `remove_date!` clears the scalar date fields *and* the edition's
+    #   year component, while preserving non-year editions/revisions (`r5`, `e2`).
+    #   Dates carried in the `update` component (some NBS supplements) are left
+    #   intact — `update` also encodes non-date update codes (`/Upd2`).
+    # - **No `(all parts)` marker in `content`.** `content` is live-derived
+    #   from `@pubid` (see below), so `to_all_parts!` freezes the already
+    #   part+date+stage-stripped rendering into `@raw_content` before
+    #   wrapping `@pubid` — matching IEC/CCSDS. `#pubid` itself, read
+    #   directly, now answers `all_parts? == true` and renders WITH pubid's
+    #   generic "(all parts)" marker, since it's the wrapper.
+    class Docidentifier < Bib::Docidentifier
+      attribute :content, :string
+
+      attr_reader :pubid
+
+      YEAR = /\A\d{4}\z/
+
+      def initialize(arg = nil, **kwargs)
+        arg.is_a?(Hash) ? super(arg) : super(**kwargs)
+        # `super` normally runs `content=` via lutaml's attribute application; the
+        # re-run is a fallback for construction paths that skip it (so the value
+        # is parsed exactly once in the common case).
+        raw = arg.is_a?(Hash) ? (arg["content"] || arg[:content]) : kwargs[:content]
+        self.content = raw if raw && @pubid.nil? && @raw_content.nil?
+      end
+
+      alias_method :original_content=, :content=
+      alias_method :original_content, :content
+
+      def content=(value)
+        @pubid = nil
+        @raw_content = nil
+
+        parsed = value.is_a?(::Pubid::Nist::Identifier) ? value : parse_pubid(value)
+
+        # Adopt the pubid only when it renders back to the exact input, so
+        # non-canonical forms (DOIs) are preserved as the raw string.
+        if parsed && (!value.is_a?(::String) || parsed.to_s(format: :human) == value)
+          @pubid = parsed
+        elsif value.is_a?(::String)
+          @raw_content = value
+        end
+
+        send(:original_content=, to_s)
+      end
+
+      def content
+        return @raw_content if @raw_content
+        return @pubid.to_s(format: :human) if @pubid
+
+        original_content
+      end
+
+      def to_s
+        content.to_s
+      end
+
+      # `#exclude` (no args) clones the whole graph (incl. a supplement's
+      # base), since `walk_chain` mutates it in place. `@raw_content` freezes
+      # the stripped rendering — `content` is live from `@pubid`, and the
+      # wrapper would otherwise add pubid's generic "(all parts)" marker,
+      # which pubid-nist itself never has. `identifiers` then keeps the
+      # ORIGINAL part/date/stage, not the stripped working copy.
+      def to_all_parts!
+        return if !@pubid || @pubid.all_parts?
+
+        original = @pubid.exclude
+        remove_part!
+        remove_date!
+        remove_stage!
+        @raw_content = to_s
+        @pubid = original.to_all_parts
+      end
+
+      def remove_part!
+        walk_chain { |node| clear_part_on(node) }
+      end
+
+      def remove_stage!
+        walk_chain { |node| node.stage = nil if node.respond_to?(:stage=) }
+      end
+
+      def remove_date!
+        walk_chain { |node| clear_date_on(node) }
+      end
+
+      private
+
+      def parse_pubid(value)
+        return nil unless value.is_a?(::String) && !value.empty?
+
+        ::Pubid::Nist::Identifier.parse(value)
+      rescue StandardError
+        Util.warn "Failed to parse Pubid: #{value}"
+        nil
+      end
+
+      # Apply a mutation to the top pubid and every identifier down the
+      # `base` chain (only SupplementIdentifier exposes the chain; a
+      # plain identifier returns nil and the walk terminates after one pass).
+      def walk_chain
+        return unless @pubid
+
+        node = @pubid
+        while node
+          yield node
+          node = node.respond_to?(:base) ? node.base : nil
+        end
+        refresh_content!
+      end
+
+      def clear_part_on(pubid)
+        pubid.part = nil if pubid.respond_to?(:part=)
+        pubid.parts = nil if pubid.respond_to?(:parts=)
+      end
+
+      # NIST spreads the date across scalar fields (mostly nil in practice) and
+      # the edition component. Clear the scalars, then strip a year that rides on
+      # the edition: a trailing year suffix (`e2.1915` -> `e2`) or a whole
+      # bare-year edition (`e1977`/`r2001` -> gone), while leaving numbered
+      # editions and revisions (`e2`, `r5`) intact.
+      def clear_date_on(pubid)
+        %i[year month revision_year revision_month edition_year update_year]
+          .each { |f| pubid.send("#{f}=", nil) if pubid.respond_to?("#{f}=") }
+
+        return unless pubid.respond_to?(:edition) && (ed = pubid.edition)
+
+        if ed.respond_to?(:additional_text) && ed.additional_text.to_s.match?(YEAR)
+          ed.additional_text = nil
+        end
+        return unless ed.respond_to?(:id) && ed.id.to_s.match?(YEAR) &&
+          (!ed.respond_to?(:additional_text) || ed.additional_text.nil?)
+
+        pubid.edition = nil if pubid.respond_to?(:edition=)
+      end
+
+      def refresh_content!
+        send(:original_content=, to_s)
+      end
+    end
+  end
+end

data/lib/relaton/nist/doctype.rb

@@ -0,0 +1,7 @@
+module Relaton
+  module Nist
+    class Doctype < Bib::Doctype
+      attribute :content, :string, values: %w[standard]
+    end
+  end
+end

data/lib/relaton/nist/ext.rb

@@ -0,0 +1,16 @@
+require_relative "doctype"
+require_relative "comment_period"
+
+module Relaton
+  module Nist
+    class Ext < Bib::Ext
+      attribute :doctype, Doctype
+      attribute :commentperiod, CommentPeriod
+
+      xml { map_element "commentperiod", to: :commentperiod }
+      key_value { map_element "commentperiod", to: :commentperiod }
+
+      def get_schema_version = Relaton.schema_versions["relaton-model-nist"]
+    end
+  end
+end

data/lib/relaton/nist/hit.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Nist
+    class Hit < Core::Hit
+      attr_writer :item
+
+      #
+      # Parse page.
+      #
+      # @return [Relaton::Nist::ItemData] bibliographic item
+      #
+      def item
+        @item ||= Scraper.parse_page @hit
+      end
+    end
+  end
+end