relaton-nist 2.1.1 → 2.2.0.pre.alpha.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a8ace15d7eadcadc014b48d587eacdd6fc15a776e774e0ea70c9de2f706bed0e
-  data.tar.gz: '08e1a24d7ca3782165e4d0d7130cbce98a77ef26b41cafb81f0d183c1947844c'
+  metadata.gz: '0280f417f1d69e2293595f8375ea938b9967746c83ac856d5239a2844300b701'
+  data.tar.gz: '0999f475da67c20b9fedee04085c477e9e7bc32c79f1753e54a460cd6425d9a7'
 SHA512:
-  metadata.gz: 9e500f1c4a9c56f755937709a4c137963226314dc72d4dcf6ef817fa91d949e09cc8fb595c394d0cde8937fde6ee53540dc949bf433559341310d111cabda2a0
-  data.tar.gz: 4466add7be3db28416eded9fc18ffdfd39c3bdc0236e529a99f48e01b1ca5220be0082429e1e4b840e7913d3d53eb125e25b3b1836c78fcb7454b4cbe06aba3f
+  metadata.gz: 8552b405a62bda69ff86fd723e02839b3e5623b07da4c9e25bb52054726f1e68de792edc6bd58bc7ecd00660c51009c70f3d3424d35136fca888c5dc6e3fb4f6
+  data.tar.gz: c08297fb05fe7b998f26c57aec6a6fae93481b26a5bd9e5d54b7cce8d3c244f4fc4bcce1e68eb4edb601a5a4181adb66eb8a666392adc15e43c26c811afe3041

data/CLAUDE.md

@@ -84,17 +84,19 @@ Tests use VCR with WebMock. Cassettes are stored in `spec/vcr_cassettes/` and re
 
 ### Test Data Stubbing
 
-Tests pre-load both the NIST index and CSRC pubs-export data from local fixtures in `before(:suite)` (see `spec/support/webmock.rb`), avoiding all HTTP requests for these data sources. VCR is configured to ignore both `index-v1.zip` and `pubs-export` requests (`spec/support/vcr.rb`).
+Tests pre-load both the NIST index and CSRC pubs-export data from local fixtures in `before(:suite)` (see `spec/support/webmock.rb`), avoiding all HTTP requests for these data sources. VCR is configured to ignore both `index-v2.zip` and `pubs-export` requests (`spec/support/vcr.rb`).
 
-- **Index**: A `Relaton::Index::Type` instance is created with `@index` set directly from `spec/fixtures/index-v1.zip`, then injected into `Relaton::Index.pool`. Run `rake spec:update_index` to refresh.
+The gem consumes the **index-v2** index (`INDEXFILE = "index-v2"`): each row's `:id` is a `Pubid::Nist::Identifier` hash (lean `to_hash` form), so `HitCollection#from_ga` narrows candidates by number via binary search before the substring block filter, and stringifies `row[:id].to_s` at the Hit boundary. `spec/fixtures/index-v2.zip` is required for the suite to run (`rake spec:update_index` downloads it from `relaton-data-nist/v2`).
+
+- **Index**: The YAML inside `spec/fixtures/index-v2.zip` is written to a temp file and loaded through `Relaton::Index::Type.new(:nist, nil, file, nil, ::Pubid::Nist::Identifier)`; calling `type.index` forces the offline read + `pubid_class` deserialization (and sort) before net is blocked. The instance is injected into `Relaton::Index.pool`, with `actual?` overridden to match only the remote (`url:`) lookup so the producer-side `find_or_create(:nist, file:, pubid_class:)` still gets a fresh instance. Run `rake spec:update_index` to refresh.
 - **PubsExport**: The `PubsExport` singleton's `@data` is set directly from `spec/fixtures/pubs-export.zip`. Run `rake spec:update_pubs_export` to refresh.
 
 To apply the index stubbing pattern to other relaton gems:
 
-1. Add `spec:update_index` rake task (downloads `index-v1.zip` from the gem's GitHub data repo)
-2. Run `rake spec:update_index` to create `spec/fixtures/index-v1.zip`
-3. In `spec/support/webmock.rb`: parse the zip once, create a `Type` instance with `@index` set directly, override `actual?` to return `true`, inject into `Relaton::Index.pool`
-4. In `spec/support/vcr.rb`: add `ignore_request` for `index-v1.zip`
+1. Add a `spec:update_index` rake task (downloads `index-v2.zip` from the gem's GitHub data repo)
+2. Run `rake spec:update_index` to create `spec/fixtures/index-v2.zip`
+3. In `spec/support/webmock.rb`: extract the zip's YAML, write it to a temp file, create a `Type` with the flavor's `pubid_class`, call `type.index` to deserialize + sort, override `actual?` to match the remote `url:` lookup, and inject into `Relaton::Index.pool`
+4. In `spec/support/vcr.rb`: add `ignore_request` for `index-v2.zip`
 5. Remove any `allow_any_instance_of(Relaton::Index::Type)` workarounds from specs
 
 ## Key Dependencies

data/Gemfile

@@ -3,6 +3,13 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in relaton_nist.gemspec
 gemspec
 
+# Use local monorepo sibling gems where available.
+Dir["../*/"].each do |dir|
+  name = File.basename(dir)
+  next if name == File.basename(__dir__)
+  next unless File.exist?(File.join(dir, "#{name}.gemspec"))
+  gem name, path: dir
+end
 
 gem "equivalent-xml", "~> 0.6"
 gem "pry-byebug"

data/README.adoc

@@ -74,7 +74,7 @@ https://github.com/relaton/loc_mods[`loc_mods`] library and its data mapped to
 the Relaton model.
 
 The NIST Library dataset provides documents listed in the
-https://github.com/relaton/relaton-data-nist/blob/main/index-v1.yaml[index].
+https://github.com/relaton/relaton-data-nist/blob/v2/index-v2.yaml[index].
 
 
 
@@ -418,7 +418,7 @@ $ rake spec:update_index
 $ rake spec:update_pubs_export
 ----
 
-`spec:update_index` downloads the latest `index-v1.zip` from the
+`spec:update_index` downloads the latest `index-v2.zip` from the
 https://github.com/relaton/relaton-data-nist[relaton-data-nist] repository.
 
 `spec:update_pubs_export` downloads the latest `pubs-export.zip` from

data/Rakefile

@@ -6,13 +6,13 @@ RSpec::Core::RakeTask.new(:spec)
 task :default => :spec
 
 namespace :spec do
-  desc "Download latest NIST index fixture from relaton-data-nist"
+  desc "Download latest NIST index fixture (index-v2) from relaton-data-nist"
   task :update_index do
     require "net/http"
     require "uri"
 
-    url = "https://raw.githubusercontent.com/relaton/relaton-data-nist/v2/index-v1.zip"
-    dest = File.join(__dir__, "spec", "fixtures", "index-v1.zip")
+    url = "https://raw.githubusercontent.com/relaton/relaton-data-nist/v2/index-v2.zip"
+    dest = File.join(__dir__, "spec", "fixtures", "index-v2.zip")
 
     puts "Downloading #{url} ..."
     uri = URI.parse(url)

data/lib/relaton/nist/bibliography.rb

@@ -13,6 +13,11 @@ module Relaton
         #
         def search(text, year = nil, opts = {})
           ref = text.sub(/^NISTIR/, "NIST IR").sub(/\/Add/, " Add")
+          # pubid 2.x only recognizes the addendum marker with a trailing
+          # period, and only splits an uppercase part letter — canonicalize
+          # both ("800-38a Add" -> "800-38A Add.") so @reference parses to the
+          # same pubid the index/CSRC carry.
+          ref = ref.sub(/\bAdd\b\.?/i, "Add.").sub(/([0-9])([a-z])(?=\s+Add\.)/) { "#{$1}#{$2.upcase}" }
           HitCollection.search ref, year, opts
         rescue OpenURI::HTTPError, SocketError, OpenSSL::SSL::SSLError => e
           raise Relaton::RequestError, e.message
@@ -86,7 +91,18 @@ module Relaton
           missed_years = []
           iteration = parse_iteration(opts[:stage])
 
-          result.each do |h|
+          # A stageless query means "the published document". Multiple
+          # editions can match once stage is excluded (e.g. a final "r2"
+          # and its draft "Rev. 2 ipd"), so try published hits before
+          # drafts. An explicit stage keeps the original order. Stable
+          # partition preserves existing tie-order among finals.
+          ordered = if iteration
+                      result.array
+                    else
+                      result.array.sort_by.with_index { |h, i| [draft_hit?(h) ? 1 : 0, i] }
+                    end
+
+          ordered.each do |h|
             r = h.item
             next if opts[:date] && !match_date?(r, opts[:date])
             next if iteration && r.status&.iteration != iteration
@@ -98,6 +114,13 @@ module Relaton
           { years: missed_years }
         end
 
+        # A hit is a draft when its CSRC status says so (e.g. "draft-public")
+        # or its rendered code carries a PD stage marker (ipd/fpd/NpD).
+        def draft_hit?(hit)
+          hit.hit[:status].to_s.include?("draft") ||
+            hit.hit[:code].to_s.match?(/(?:\bi|\bf|\b\d)pd\b|\(Draft\)/i)
+        end
+
         def parse_iteration(stage)
           iter = /\w+(?=PD)|(?<=PD-)\w+/.match(stage)&.to_s
           case iter

data/lib/relaton/nist/data_fetcher.rb

@@ -29,13 +29,24 @@ module Relaton
         id = bib.docidentifier.find(&:primary) || bib.docidentifier.first
         file = output_file id.content.sub(/^NIST IR/, "NISTIR")
         if @files.include? file
-          Util.warn "File #{file} exists. Docid: #{bib.docidentifier[0].content}"
+          Util.warn "File #{file} exists. Docid: #{id.content}"
         else @files << file
         end
-        index.add_or_update bib.docidentifier[0].content, file
+        pid = pubid id.content
+        index.add_or_update pid, file if pid
         File.write file, serialize(bib), encoding: "UTF-8"
       end
 
+      # Parse a docidentifier string into a Pubid::Nist::Identifier; nil (with a
+      # warning) if pubid can't parse it, so a single bad id never aborts the
+      # crawl or corrupts index-v2.
+      def pubid(id)
+        ::Pubid::Nist::Identifier.parse id
+      rescue StandardError => e
+        Util.warn "Failed to parse `#{id}` with pubid: #{e.message}"
+        nil
+      end
+
       # def add_static_files
       #   Dir["./static/*.yaml"].each do |file|
       #     bib = Item.from_yaml(File.read(file, encoding: "UTF-8"))
@@ -60,7 +71,9 @@ module Relaton
       end
 
       def index
-        @index ||= Relaton::Index.find_or_create :nist, file: "#{INDEXFILE}.yaml"
+        @index ||= Relaton::Index.find_or_create(
+          :nist, file: "#{INDEXFILE}.yaml", pubid_class: ::Pubid::Nist::Identifier
+        )
       end
 
       def series

data/lib/relaton/nist/hit_collection.rb

@@ -61,7 +61,7 @@ module Relaton
         refid = ::Pubid::Nist::Identifier.parse(@reference)
         parts = exclude_parts refid
         arr = @array.select do |item|
-          pubid = ::Pubid::Nist::Identifier.parse(item.hit[:code].sub(/\.$/, ""))
+          pubid = ::Pubid::Nist::Identifier.parse(item.hit[:code])
           pubid.exclude(*parts) == refid
         rescue StandardError
           item.hit[:code] == ref
@@ -74,10 +74,31 @@ module Relaton
         return dup
       end
 
+      # The edition/revision/version attributes that together identify a
+      # specific edition of a document. Treated as one unit when deciding
+      # whether a reference is "incomplete" (no edition given).
+      EDITION_FAMILY = %i[
+        edition edition_component revision revision_year revision_month
+        version version_component edition_year
+      ].freeze
+
       def exclude_parts(pubid)
-        %i[stage update].each_with_object([]) do |part, parts|
-          parts << part if pubid.send(part).nil?
+        # Pubid 2.x exposes update via two slots (:update and :update_component),
+        # and pubs_export_id assigns to update_component — so checking only
+        # :update misses the case where the indexed pubid carries the update
+        # info there. Same logic for both legacy and component slots.
+        parts = %i[stage update update_component].select do |part|
+          pubid.respond_to?(part) && pubid.send(part).nil?
+        end
+
+        # Incomplete reference: no edition/revision/version specified (e.g.
+        # "NIST SP 800-60v1"). Exclude the whole edition family so it matches
+        # any edition; result selection (sort_hits! + results_filter) then
+        # picks the latest/preferred one.
+        if EDITION_FAMILY.all? { |p| !pubid.respond_to?(p) || pubid.send(p).nil? }
+          parts += EDITION_FAMILY
         end
+        parts
       end
 
       private
@@ -208,13 +229,31 @@ module Relaton
       #
       def from_ga # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
         Util.info "Fetching from Relaton repository ...", key: @reference
-        r = pubid
+        # Parse the reference so index.search narrows candidates by number via
+        # binary search; fall back to the raw string for index lookup if pubid
+        # can't parse it.
+        r = begin
+          ::Pubid::Nist::Identifier.parse(@reference)
+        rescue StandardError
+          @reference
+        end
 
-        index = Relaton::Index.find_or_create :nist, url: "#{GHNISTDATA}index-v1.zip", file: "#{INDEXFILE}.yaml"
-        rows = index.search(r).sort_by { |row| row[:id] }
+        index = Relaton::Index.find_or_create(
+          :nist, url: "#{GHNISTDATA}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml",
+          pubid_class: ::Pubid::Nist::Identifier
+        )
+        # search(r) narrows candidates by number via binary search; the block
+        # keeps the broad family match (this doc and all its editions/parts)
+        # the old string index gave via substring, leaving finer filtering to
+        # search_filter.
+        needle = r.to_s
+        rows = index.search(r) { |row| row[:id].to_s.include?(needle) }
+          .sort_by { |row| row[:id].to_s }
 
         rows.map do |row|
-          Hit.new({ code: row[:id], path: row[:file] }, self)
+          # index-v2 stores Pubid objects; the rest of the pipeline (search_filter,
+          # Hit, Scraper) works on string codes, so stringify at the boundary.
+          Hit.new({ code: row[:id].to_s, path: row[:file] }, self)
         end
       rescue OpenURI::HTTPError => e
         return [] if e.io.status[0] == "404"
@@ -246,8 +285,19 @@ module Relaton
         end => id
 
         id.sub!(/(?:-draft\d*|\.\wpd)$/, "")
-        id = id.gsub(".", " ").sub(/-Add(\d*)$/, ' Add\1') if id.match?(/-Add\d*$/)
-        pid = ::Pubid::Nist::Identifier.parse(id)
+        id = id.gsub(".", " ").sub(/-Add(\d*)$/, ' Add.\1') if id.match?(/-Add\d*$/)
+        # Normalize to space-separated form so pubid 2.x parses as
+        # parsed_format=:short and renders without dots; also force the
+        # "NIST " prefix so publisher_was_parsed is set.
+        parse_input = id.gsub(/\bNIST\./, "NIST ")
+        parse_input = "NIST #{parse_input}" unless parse_input.start_with?("NIST ")
+        pid = ::Pubid::Nist::Identifier.parse(parse_input)
+
+        # Canonicalize edition spelling: DOI-derived codes are already short
+        # ("…r2"), but no-DOI drafts come from the verbose docidentifier
+        # ("… Rev. 2") and pubid preserves that spelling via original_prefix.
+        # Drop it so every code renders the canonical short form.
+        pid.edition.original_prefix = nil if pid.respond_to?(:edition) && pid.edition
 
         # Stage: URI is authoritative, fall back to iteration. "final" => no stage.
         uri_stage = json["uri"] && json["uri"][%r{/(final|ipd|fpd|\dpd)(?:-\(\d+\))?(?:/|$)}, 1]
@@ -256,13 +306,17 @@ module Relaton
         when nil, "final"
           # no stage — "final" means published
         when "fpd"
-          pid.stage = ::Pubid::Nist::Stage.new id: "f", type: "pd"
+          pid.stage = ::Pubid::Nist::Components::Stage.new id: "f", type: "pd"
         when /\A(\w)pd\z/
-          pid.stage = ::Pubid::Nist::Stage.new id: Regexp.last_match(1), type: "pd"
+          pid.stage = ::Pubid::Nist::Components::Stage.new id: Regexp.last_match(1), type: "pd"
         end
 
         /\/upd(?<upd>\d+)\// =~ json["uri"]
-        pid.update = Pubid::Nist::Update.new number: upd if upd
+        # In pubid 2.x render paths the Update is read from
+        # update_component (Components::Update), not the legacy :update
+        # slot — assigning to :update would render as "-upd/Upd2" via the
+        # elsif fallback path in Base#to_short_style.
+        pid.update_component = ::Pubid::Nist::Components::Update.new(number: upd) if upd
         pid.to_s
       rescue StandardError
         id += " #{json["iteration"]}" if json["iteration"] && json["iteration"] != "final"

data/lib/relaton/nist/mods_parser.rb

@@ -48,8 +48,14 @@ module Relaton
       def get_id_from_str(str)
         return if str.nil? || str.empty?
 
-        ::Pubid::Nist::Identifier.parse(str).to_s
-      rescue ::Pubid::Core::Errors::ParseError
+        # DOIs arrive dotted ("NIST.HB.135e2022-upd1"); that MR-style form
+        # parses through a path that skips UpdateCodes date lookup and drops
+        # letter suffixes. Space the leading "PUB.SERIES." prefix so it parses
+        # like the canonical short form (only the separator dots are touched,
+        # not any inside the number). Then force :human rendering.
+        spaced = str.sub(/\A(NIST|NBS)\.([A-Z][A-Za-z]*)\./, '\1 \2 ')
+        ::Pubid::Nist::Identifier.parse(spaced).to_s(format: :human)
+      rescue Parslet::ParseFailed
         str.gsub(".", " ").sub(/^[\D]+/, &:upcase)
       end
 

data/lib/relaton/nist/processor.rb

@@ -59,7 +59,9 @@ module Relaton
       #
       def remove_index_file
         require_relative "../nist"
-        Relaton::Index.find_or_create(:nist, url: true, file: "#{INDEXFILE}.yaml").remove_file
+        Relaton::Index.find_or_create(
+          :nist, url: true, file: "#{INDEXFILE}.yaml", pubid_class: ::Pubid::Nist::Identifier
+        ).remove_file
       end
     end
   end

data/lib/relaton/nist/version.rb

@@ -1,5 +1,5 @@
 module Relaton
   module Nist
-    VERSION = "2.1.1".freeze
+    VERSION = "2.2.0.pre.alpha.1".freeze
   end
 end

data/lib/relaton/nist.rb

@@ -17,7 +17,11 @@ require_relative "nist/bibliography"
 
 module Relaton
   module Nist
-    INDEXFILE = "index-v1"
+    # Pubid-based index (index-v2): `:id` is a Pubid::Nist::Identifier hash, so
+    # search narrows by number via binary search. The legacy string index-v1
+    # (for older gem versions) is rebuilt separately by the relaton-data-nist
+    # crawler, not by this gem's DataFetcher.
+    INDEXFILE = "index-v2"
 
     class Error < StandardError; end
 

data/relaton-nist.gemspec

@@ -21,14 +21,14 @@ Gem::Specification.new do |spec|
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
-  spec.required_ruby_version = Gem::Requirement.new(">= 3.2.0")
+  spec.required_ruby_version = Gem::Requirement.new(">= 3.3.0")
 
   spec.add_dependency "base64"
   spec.add_dependency "mechanize", "~> 2.0"
   spec.add_dependency "loc_mods", "~> 0.3.0"
-  spec.add_dependency "pubid", "~> 1.15.6"
-  spec.add_dependency "relaton-bib", "~> 2.1.0"
-  spec.add_dependency "relaton-core", "~> 0.0.13"
-  spec.add_dependency "relaton-index", "~> 0.2.0"
+  spec.add_dependency "pubid", "~> 2.0.0.pre.alpha.3"
+  spec.add_dependency "relaton-bib", "~> 2.2.0.pre.alpha.1"
+  spec.add_dependency "relaton-core", "~> 2.2.0.pre.alpha.1"
+  spec.add_dependency "relaton-index", "~> 2.2.0.pre.alpha.1"
   spec.add_dependency "rubyzip"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-nist
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 2.2.0.pre.alpha.1
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-05-12 00:00:00.000000000 Z
+date: 2026-06-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base64
@@ -58,56 +58,56 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.15.6
+        version: 2.0.0.pre.alpha.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.15.6
+        version: 2.0.0.pre.alpha.3
 - !ruby/object:Gem::Dependency
   name: relaton-bib
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: 2.2.0.pre.alpha.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: 2.2.0.pre.alpha.1
 - !ruby/object:Gem::Dependency
   name: relaton-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.13
+        version: 2.2.0.pre.alpha.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.13
+        version: 2.2.0.pre.alpha.1
 - !ruby/object:Gem::Dependency
   name: relaton-index
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 2.2.0.pre.alpha.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 2.2.0.pre.alpha.1
 - !ruby/object:Gem::Dependency
   name: rubyzip
   requirement: !ruby/object:Gem::Requirement
@@ -133,7 +133,6 @@ files:
 - ".github/workflows/release.yml"
 - ".gitignore"
 - ".rspec"
-- ".rubocop.yml"
 - CLAUDE.md
 - Gemfile
 - LICENSE.txt
@@ -142,11 +141,6 @@ files:
 - bin/console
 - bin/rspec
 - bin/setup
-- grammars/basicdoc.rng
-- grammars/biblio-standoc.rng
-- grammars/biblio.rng
-- grammars/relaton-nist-compile.rng
-- grammars/relaton-nist.rng
 - lib/relaton/nist.rb
 - lib/relaton/nist/bibdata.rb
 - lib/relaton/nist/bibitem.rb
@@ -181,7 +175,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.2.0
+      version: 3.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="

data/.rubocop.yml

@@ -1,12 +0,0 @@
-# This project follows the Ribose OSS style guide.
-# https://github.com/riboseinc/oss-guides
-# All project-specific additions and overrides should be specified in this file.
-
-require: rubocop-rails
-
-inherit_from:
-  - https://raw.githubusercontent.com/riboseinc/oss-guides/master/ci/rubocop.yml
-AllCops:
-  TargetRubyVersion: 3.2
-Rails:
-  Enabled: false