relaton 3.0.0 → 3.0.2

data/lib/relaton/ietf/data_fetcher.rb

@@ -216,41 +216,14 @@ module Relaton
       end
 
       #
-      # Read a bibxml file as UTF-8, recovering Windows-1252 bytes.
-      #
-      # These files declare `encoding='UTF-8'` but some carry CP1252 — smart
-      # quotes and accented Latin letters. `File.read(encoding: "UTF-8")` only
-      # *tags* the string, so those reach lutaml as invalid UTF-8 and it raises.
-      #
-      # `scrub` with a block transcodes each invalid *run* as CP1252 while
-      # leaving valid UTF-8 untouched. Both halves matter:
-      #
-      # * Not plain `scrub`, which substitutes U+FFFD: `client’s` would become
-      #   `client\uFFFDs` and `Muñoz` `Mu\uFFFDoz` — author surnames included.
-      #   The damage is length-preserving, so a length check will not catch it.
-      # * Not a whole-file CP1252 re-decode, which mangles a file that is
-      #   genuinely UTF-8 apart from one stray byte: `Muñoz café ’` would come
-      #   back as `Muñoz café ’`, silently, since the result is valid UTF-8 and
-      #   nothing raises. No such file exists in today's corpus (0 of the 125
-      #   affected contain valid multi-byte UTF-8) but it grows daily, and
-      #   "decodes losslessly as CP1252" is weak evidence of correctness —
-      #   CP1252 maps 251 of 256 byte values.
-      #
-      # `undef: :replace` covers the five bytes CP1252 leaves undefined
-      # (0x81 0x8D 0x8F 0x90 0x9D), so this returns valid UTF-8 rather than
-      # raising and costing the whole file.
+      # Read a bibxml file as UTF-8, recovering Windows-1252 bytes. The repair
+      # is `BibXMLParser.to_utf8`, shared with `Scraper`'s bib.ietf.org fallback.
       #
       # @param path [String]
       # @return [String] UTF-8, valid encoding
       #
       def read_bibxml(path)
-        utf8 = File.binread(path).force_encoding(Encoding::UTF_8)
-        return utf8 if utf8.valid_encoding?
-
-        utf8.scrub do |bad|
-          bad.force_encoding(Encoding::WINDOWS_1252)
-             .encode(Encoding::UTF_8, undef: :replace)
-        end
+        BibXMLParser.to_utf8 File.binread(path)
       end
 
       #

data/lib/relaton/ietf/scraper.rb

@@ -2,6 +2,7 @@
 
 require "pubid"
 require "pubid/ietf"
+require_relative "bibxml_parser"
 
 module Relaton
   module Ietf
@@ -15,6 +16,13 @@ module Relaton
       # publishing their `index-v1` for released relatons, untouched.
       IETF = "https://raw.githubusercontent.com/relaton/relaton-data-ietf/main/"
 
+      # The IETF bibxml service, read when the index has no row for a versioned
+      # Internet-Draft (relaton-ietf#125). The data repo crawls the rsync
+      # `bibxml-ids` snapshot, which keeps only some versions of each draft
+      # (`draft-ietf-vcarddav-vcardrev`: only -22); this service resolves any
+      # version the datatracker knows, and answers 404 for one it does not.
+      BIBXML_IDS = "https://bib.ietf.org/public/rfc/bibxml-ids/"
+
       # @param ref [String, Pubid::Ietf::Identifier] a reference, or its parse
       #   from Relaton::Db (relaton#205), which is used as it is
       # @return [Relaton::Ietf::ItemData, nil]
@@ -24,8 +32,9 @@ module Relaton
 
         fetch_doc id
       rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError,
-             Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
-             Net::ProtocolError, SocketError
+             Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH,
+             OpenSSL::SSL::SSLError, Net::HTTPBadResponse,
+             Net::HTTPHeaderSyntaxError, Net::ProtocolError, SocketError
         raise Relaton::RequestError, "No document found for #{ref} reference"
       end
 
@@ -44,7 +53,55 @@ module Relaton
       # such pairs in the first 40k rows of the index fixture).
       def fetch_doc(id)
         row = index.search(id, exact: true).first
-        get_page "#{IETF}#{row[:file]}" if row
+        return get_page "#{IETF}#{row[:file]}" if row
+
+        fetch_bibxml id if versioned_draft? id
+      end
+
+      # Only a versioned draft falls back. The index holds every RFC, sub-series
+      # and unversioned draft aggregator the data repo has, so a miss on one of
+      # those is a real "not found", and bib.ietf.org has no aggregator to offer.
+      def versioned_draft?(id)
+        id.is_a?(::Pubid::Ietf::Identifiers::InternetDraft) && !id.version.to_s.empty?
+      end
+
+      #
+      # Read one draft version from bib.ietf.org and parse it the way
+      # `DataFetcher#process_singleton` parses a crawled file: the same
+      # CP1252 repair, the same parser, and the version from the identifier.
+      #
+      # A 404 is the service's "no such version". Any other non-200 (a 429, a
+      # 5xx) is push-back or an outage, so it raises `Relaton::RequestError`:
+      # `Db#net_retry` then retries it, and the Db cache does not store it as a
+      # not-found.
+      #
+      # @param id [Pubid::Ietf::Identifiers::InternetDraft]
+      # @return [Relaton::Ietf::ItemData, nil] nil on a 404 or an unreadable body
+      #
+      def fetch_bibxml(id)
+        res = Net::HTTP.get_response URI("#{BIBXML_IDS}reference.I-D.#{id}.xml")
+        return if res.code == "404"
+        raise Relaton::RequestError, "#{BIBXML_IDS} answered #{res.code} for #{id}" unless res.code == "200"
+
+        bib = parse_bibxml(res.body, id)
+        return unless bib
+
+        bib.version = [Bib::Version.new(draft: id.version.to_s)]
+        bib.fetched = Date.today.to_s
+        bib
+      end
+
+      # A 200 that is not bibxml (an HTML error page, a truncated body) costs
+      # this one document, not the caller's compile. These are the errors the
+      # parse raises on such a body: lutaml's `InvalidFormatError`, and the
+      # converter's `NoMethodError` on a document with no `<front>` and
+      # `ArgumentError` from its regexes. They are named, not `StandardError`,
+      # so a transport error still reaches `scrape_page`'s rescue.
+      def parse_bibxml(body, id)
+        BibXMLParser.parse BibXMLParser.to_utf8(body)
+      rescue Lutaml::Model::InvalidFormatError, NoMethodError, ArgumentError => e
+        Util.warn "Unreadable bibxml for `#{id}`: #{e.message.to_s.lines.first.to_s.strip}"
+        nil
       end
 
       #

data/lib/relaton/ietf/wg_name_resolver.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "json"
+
 require "net/http"
 
 module Relaton
@@ -20,7 +20,7 @@ module Relaton
           response = Net::HTTP.get_response(uri)
           break unless response.is_a?(Net::HTTPSuccess)
 
-          data = JSON.parse(response.body)
+          data = Relaton::Json.parse_external(response.body)
           objects = data["objects"] || []
           break if objects.empty?
 

data/lib/relaton/iho/CLAUDE.md

@@ -55,6 +55,10 @@ In `item.rb`, `require_relative "relation"` is placed **after** the `Item` class
 
 `Bibliography.search`/`.get` fetches YAML from the relaton-data-iho GitHub repo via `Relaton::Index`, then deserializes into model objects.
 
+**Editions.** The pubid `version` is the edition (`IHO B-12 3.0.0`; the grammar takes `a.b.c` only). A reference with no version answers with the **latest** edition: `version_key` compares the segments as integers (`2.10.0` after `2.9.0`). It used to be `min_by` on the string, which returned the oldest.
+
+**The edition reaches the index only through the committed row.** A record prints `B-12` and keeps its edition in `edition:`, so `relaton index` cannot read a version from the docid. `MachineIndex#refine` (relaton-cli) takes the committed `index-v3.yaml` row when it adds only a `version` to the parse. Before that, the Pages index (generated 2026-10-29) had no `version` in any of its 325 rows, so every versioned reference answered not-found (relaton-data-iho#20). The suite did not see it: `spec/fixtures/index-v3.zip` is the committed index, which has versions.
+
 ### Test Infrastructure
 
 - **VCR cassettes** (`spec/vcr_cassettes/`): recorded HTTP responses for offline testing

data/lib/relaton/iho/bibliography.rb

@@ -21,7 +21,7 @@ module Relaton
           # Pubid::Iho::Identifiers::* (Relaton::Index deserialized it via the
           # `pubid_class` passed in `#index`), so `pubid_match?` compares pubids.
           row = index.search(pubid) { |r| pubid_match?(r[:id], pubid) }
-                     .min_by { |r| r[:id].version.to_s }
+                     .max_by { |r| version_key(r[:id]) }
           unless row
             Util.info "Not found.", key: pubid.to_s
             return
@@ -96,6 +96,12 @@ module Relaton
           )
         end
 
+        # The sort key for the latest edition: segments compare as integers,
+        # so 2.10.0 is later than 2.9.0. A row with no version is the oldest.
+        def version_key(id)
+          id.version.to_s.split(".").map(&:to_i)
+        end
+
         # Both `row_id` and `query` are Pubid::Iho::Identifiers::* instances.
         # Class identity pins the series (S/P/M/B/C). Subdivision keys
         # (part/appendix/annex/supplement) use strict equality — a nil query

data/lib/relaton/index/dataset_lock.rb

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
 
+require "yeptris"
+
 require "digest"
-require "json"
 require "net/http"
 require "time"
 require "uri"
@@ -91,8 +92,8 @@ module Relaton
         end
         raise Error, "cannot fetch #{uri}: HTTP #{res.code}" unless res.code == "200"
 
-        JSON.parse(res.body)
-      rescue JSON::ParserError => e
+        Yeptris::JSON.load(res.body)
+      rescue Yeptris::JSON::ParseError => e
         raise Error, "malformed manifest at #{uri}: #{e.message}"
       end
 

data/lib/relaton/index/shard_source.rb

@@ -1,3 +1,4 @@
+require "yeptris"
 require "json"
 require "net/http"
 require "openssl"
@@ -174,8 +175,8 @@ module Relaton
       end
 
       def parse_json(body, path)
-        JSON.parse(body)
-      rescue JSON::ParserError => e
+        Yeptris::JSON.load(body)
+      rescue Yeptris::JSON::ParseError => e
         raise Error, "cannot parse `#{url(path)}`: #{e.message}"
       end
 

data/lib/relaton/index/sqlite_backend.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "yeptris"
+
 require "sqlite3"
 
 module Relaton
@@ -34,7 +36,7 @@ module Relaton
         @db.transaction do
           @db.execute("DELETE FROM index_rows")
           @db.prepare("INSERT INTO index_rows (number, sort_key, id_json, file) VALUES (?, ?, ?, ?)") do |stmt|
-            rows_enum.each { |(number, sort_key, id_hash, file)| stmt.execute(number, sort_key, JSON.generate(id_hash), file) }
+            rows_enum.each { |(number, sort_key, id_hash, file)| stmt.execute(number, sort_key, Yeptris::JSON.dump(id_hash), file) }
           end
           meta_set("schema_version", SCHEMA_VERSION)
         end
@@ -44,14 +46,14 @@ module Relaton
       def bucket(number)
         rows = []
         @db.execute("SELECT id_json, file FROM index_rows WHERE number = ? ORDER BY rowid", [number]) do |row|
-          rows << { id: JSON.parse(row[0]), file: row[1] }
+          rows << { id: Yeptris::JSON.load(row[0]), file: row[1] }
         end
         rows
       end
 
       def each_row
         @db.execute("SELECT id_json, file FROM index_rows ORDER BY rowid") do |row|
-          yield({ id: JSON.parse(row[0]), file: row[1] })
+          yield({ id: Yeptris::JSON.load(row[0]), file: row[1] })
         end
       end
 

data/lib/relaton/isbn/open_library.rb

@@ -1,3 +1,4 @@
+require "yeptris"
 module Relaton
   module Isbn
     #
@@ -37,7 +38,7 @@ module Relaton
         response = Net::HTTP.get_response uri
         return unless response.is_a? Net::HTTPSuccess
 
-        data = JSON.parse response.body
+        data = Relaton::Json.parse_external response.body
         return unless data["records"]&.any?
 
         data["records"].first.last

data/lib/relaton/iso/data_fetcher.rb

@@ -1,5 +1,5 @@
+require "yeptris"
 require "fileutils"
-require "json"
 require "net/http"
 require "tmpdir"
 require_relative "../iso"
@@ -168,7 +168,7 @@ module Relaton
         amend_map = Hash.new { |h, k| h[k] = [] }
         date_map = {}
         File.foreach(path, encoding: "UTF-8") do |line|
-          rec = JSON.parse(line)
+          rec = Relaton::Json.parse_external(line)
           id = rec["id"]
           ref = normalize_reference(rec["reference"])
           next unless ref
@@ -179,7 +179,7 @@ module Relaton
           if rec["supplementType"] && (base = amend_base(ref))
             amend_map[base] << ref
           end
-        rescue JSON::ParserError
+        rescue ::JSON::ParserError
           next
         end
         Util.info "Indexed #{ref_map.size} references; " \
@@ -216,11 +216,11 @@ module Relaton
         path = download_tc_dataset
         map = {}
         File.foreach(path, encoding: "UTF-8") do |line|
-          rec = JSON.parse(line)
+          rec = Relaton::Json.parse_external(line)
           ref = rec["reference"]
           title = rec["title"]
           map[ref] = title if ref && title.is_a?(Hash)
-        rescue JSON::ParserError
+        rescue ::JSON::ParserError
           next
         end
         Util.info "Indexed #{map.size} committees."
@@ -231,7 +231,7 @@ module Relaton
         Util.info "Parsing records..."
         count = 0
         File.foreach(path, encoding: "UTF-8") do |line|
-          rec = JSON.parse(line)
+          rec = Relaton::Json.parse_external(line)
           next unless ingestable?(rec["reference"])
 
           fetch_pub(rec, ref_index, tc_index, amend_index, date_index)

data/lib/relaton/iso/model/docidentifier.rb

@@ -64,10 +64,6 @@ module Relaton
         original_content
       end
 
-      def to_s
-        content.to_s
-      end
-
       # `remove_part!`/`remove_date!` are unnecessary here: `to_all_parts`'s
       # own identity computation already excludes part/date (down the whole
       # base chain, so a supplement's own year is dropped too — a disclosed

data/lib/relaton/itu/data_fetcher.rb

@@ -1,5 +1,5 @@
+require "yeptris"
 require "net/http"
-require "json"
 require "uri"
 require "mechanize"
 require_relative "../itu"
@@ -542,7 +542,7 @@ module Relaton
         request["User-Agent"] = USER_AGENT
 
         response = http.request(request)
-        json = JSON.parse(response.body)
+        json = Relaton::Json.parse_external(response.body)
         json["Data"] || []
       end
     end

data/lib/relaton/itu/hit_collection.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
+
 require "cgi"
-require "json"
 require_relative "hit"
 
 module Relaton
@@ -239,8 +239,8 @@ module Relaton
       # @return [Array<Hash>] editions of the recommendation
       def editions(idrec)
         # a `null` body parses to nil, so `|| []` is not redundant
-        JSON.parse(agent.get(format(RECEDITIONS_URL, idrec: idrec)).body) || []
-      rescue JSON::ParserError
+        Relaton::Json.parse_external(agent.get(format(RECEDITIONS_URL, idrec: idrec)).body) || []
+      rescue ::JSON::ParserError
         []
       end
 

data/lib/relaton/itu/recommendation_fields.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "json"
+
 require_relative "family_cache"
 
 module Relaton
@@ -251,7 +251,7 @@ module Relaton
       end
 
       def get_data(url)
-        JSON.parse request_document(url).body
+        Relaton::Json.parse_external request_document(url).body
       end
 
       def request_document(url)

data/lib/relaton/jis/data_fetcher.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+
 require_relative "../jis"
 require_relative "scraper"
 
@@ -92,7 +93,7 @@ module Relaton
         body = { record: 0, dantai: "JIS", searchtype2: 1,
                  status_1: 1, status_2: 2 } # rubocop:disable Naming/VariableNumber
         resp = agent.post "#{URL}W11M0270/index", body
-        disp = JSON.parse resp.body
+        disp = Relaton::Json.parse_external resp.body
         @initial_time = Time.now
         disp["status"] || Util.warn("No results found for JIS")
       end

data/lib/relaton/jis/hit_collection.rb

@@ -95,8 +95,7 @@ module Relaton
       def index
         @index ||= Relaton::Index.find_or_create(
           :jis,
-          url: "#{GH_URL}#{INDEXFILE}.zip",
-          file: "#{INDEXFILE}.yaml",
+          pages_url: PAGES_URL,
           pubid_class: ::Pubid::Jis::Identifier,
         )
       end

data/lib/relaton/jis.rb

@@ -17,6 +17,7 @@ module Relaton
   module Jis
     # The pubid index. relaton-data-jis rebuilds the legacy index-v1 from data/.
     INDEXFILE = "index-v2"
+    PAGES_URL = "https://relaton.github.io/relaton-data-jis/"
 
     class Error < StandardError; end
 

data/lib/relaton/json.rb

@@ -0,0 +1,21 @@
+require "yeptris"
+
+module Relaton
+  # JSON at the system boundary — data we do not control. Everything we
+  # generate or own (index manifests, shards, the db cache, request bodies,
+  # the schema-version table) parses with Yeptris::JSON's default strict
+  # surface.
+  module Json
+    module_function
+
+    # Third-party feeds carry duplicate keys in the wild (Crossref book
+    # records keyed by ISBN, NIST's scraped pubs export keyed by "&nbsp",
+    # the UN search API's blank keys, CCSDS's embedded config); parsing them
+    # the historical way is last-wins. yeptris 0.6.51 carries the boundary
+    # opt-out (yeptris-ruby#335/#339), so the whole stack stays on yeptris —
+    # no stdlib fallback anywhere.
+    def parse_external(source)
+      ::Yeptris::JSON.load(source, allow_duplicate_key: true)
+    end
+  end
+end

data/lib/relaton/logger/channels/gh_issue.rb

@@ -1,5 +1,5 @@
+require "yeptris"
 require "net/http"
-require "json"
 
 module Relaton
   module Logger
@@ -70,7 +70,7 @@ module Relaton
           responce = get(uri)
           return nil unless responce.code.to_i == 200
 
-          items = JSON.parse(responce.body)["items"] || []
+          items = Yeptris::JSON.load(responce.body)["items"] || []
           items.find { |i| i["title"] == @title }&.fetch("number", nil)
         rescue StandardError => e
           puts "Failed to search for an open `#{@title}` issue: #{e.message}"
@@ -88,7 +88,7 @@ module Relaton
           http.use_ssl = true
 
           request = Net::HTTP::Post.new(uri.request_uri, headers)
-          request.body = body.to_json
+          request.body = Yeptris::JSON.dump(body)
 
           http.request(request)
         end

data/lib/relaton/logger/formatter_json.rb

@@ -1,8 +1,9 @@
+require "yeptris"
 module Relaton::Logger
   class FormatterJSON < ::Logger::Formatter
     def call(severity, _datetime, progname, msg, **args)
       hash = { prog: progname, message: msg, severity: severity }.merge(args)
-      "#{hash.to_json}\n"
+      Yeptris::JSON.dump(hash) + "\n"
     end
   end
 end

data/lib/relaton/nist/CLAUDE.md

@@ -64,6 +64,7 @@ All data models inherit from `Lutaml::Model::Serializable` and use declarative a
   - **Per-record resilience (mirrors the ISO flavor).** A single bad record must never abort the crawl. `#write_file` skips (never `id.content`-crashes on) a record whose MODS entry yields **no docidentifier** (e.g. a DOI stored only in `<identifier type="doi">`, which `ModsParser#parse_doi` doesn't read), and records ids that `#pubid` can't parse (new series like `NIST RB`/`NIST CHIPS`) as "not indexed" — the YAML is still written, only the index-v2 entry is skipped. Both classes accumulate in `#failures`; the `#report_errors` override emits each at `:error` (via `#log_error` → `Util.error`), which the shared `Core::DataFetcher` gh_issue machinery turns into one "Error fetching documents" GitHub issue in CI (`GITHUB_REPOSITORY` + `GITHUB_TOKEN`). `#pubid` no longer warns per-failure; reporting is centralized in `#report_errors`.
   - **Draft-stage rows come from CSRC (relaton#313).** The MODS collection has no draft records, so `#fetch_csrc` adds the CSRC pubs-export feed (`PubsExport.instance.data`) after the MODS pass. It builds each row **exactly as the runtime builds a CSRC hit** — `HitCollection#hit_data` (which calls `#pubs_export_id`) → `Scraper.parse_page` — so the crawled id and the live id are one string, with the stage, revision and update from the row's uri and DOI (`NIST FIPS 197/Upd1 ipd`, `NIST SP 800-108r1 ipd`, `NIST SP 1800-12 2pd`). Only a row whose id carries a **stage** is written; finals come from MODS, so a CSRC row never shadows a base document. On the 914-row fixture feed this writes 302 staged records. Don't go back to a hand-built `"NIST #{series} #{docnumber} #{stage}"` id: it dropped the numbered drafts (`2pd`…), the `/UpdN`, and spelled revisions `Rev. 1`. A stage that CSRC does not list (e.g. `NIST FIPS 197 fpd`) still answers through the consumer's #303 family fallback. Three more rules: `#csrc_bib` clears the `fetched` date the runtime stamps, or every draft changes each day and `Core#force_write` rewrites ~300 files per incremental crawl; CSRC lists a republished draft twice (`…/ipd` and `…/ipd-(1)`, one DOI, so one id), so each id is written once and the plain uri wins; and only the build step is rescued (to a `#failures` entry) — a write error stops the crawl, as in the MODS pass. `#pubs_export_id` must not mutate its row (`sub`, not `sub!`): the crawler and the runtime share the `PubsExport` data.
   - **A CSRC relation names the related document (relaton#314).** `Scraper#related_id` finds the related feed row by its uri and derives its id with `HitCollection.pubs_export_id` (the class method the instance method delegates to). The entry's own `docidentifier` drops the update and the part, and the hit's code is the record's own id, so neither is the related id. `HitCollection#search_filter` ends in an `ensure` that returns `arr`: every branch must set `arr`, because an earlier `return` is overridden there.
+  - **CSWP comes from CSRC, not the NIST Library (relaton-nist#104).** The NIST Library announced CSWP data, but its MODS release (`allrecords-MODS.xml`, and the MARC `allrecords.xml`) holds **no** CSWP record — 0 on the Sept2026 release. It publishes CSWPs only as RIS/BibTeX files under `bib/` on its `nist-pages` branch. So the index has only the CSRC draft CSWP rows (`#fetch_csrc`), and a final CSWP answers from CSRC at runtime: `HitCollection#search` asks the CSRC feed (`from_json`) and asks the index (`from_ga`) only when CSRC gives no hit. That is the chosen rule — CSRC wins, no merge — because CSRC lists every numbered CSWP with more metadata (affiliations, status, supersedes, comment period). The known gap: the NIST Library also has about 35 **legacy dated ids** (`NIST CSWP 01142020`, the same paper as `CSWP 9`, from before NIST renumbered the series) and translations (`NIST.CSWP.6.spa`). CSRC lists neither, so they answer not-found; a fix needs a mapping built from those RIS files. `hit_collection_spec.rb` pins the CSRC answer.
   - **`#write_file` serializes before it indexes**, so a record that fails to serialize leaves no index row naming a file that was never written. That is how a `Bib::Date` in `Scraper.fetch_dates` hid: `Nist::ItemData` takes `Nist::Date`, and YAML serialization rejects the parent class (the same strictness as the docidentifier note below).
 - `ModsParser` — maps MODS XML (via `loc_mods`) to `ItemData`
 - `Scraper` — fetches items from GitHub YAML or CSRC JSON
@@ -118,6 +119,11 @@ NIST-specific gotchas (why this isn't a straight copy of IEC/CCSDS):
   `@raw_content` before wrapping `@pubid` in pubid's `AllParts` — matching
   IEC/CCSDS. `#pubid` itself, read directly, now answers `all_parts? ==
   true` and renders WITH pubid's generic "(all parts)" marker.
+- **The internal form drops `NIST `/`NBS `** (relaton#49), so
+  `docid.to_s(form: :internal)` is `SP 800-113` for `NIST SP 800-113`. The rule
+  is the `"NIST"` row of `Bib::Docidentifier::FORM_PUBID`: `@pubid`
+  renders with `to_s(with_publisher: false)`. A DOI docid has type `DOI` and is
+  unchanged. `to_s` itself is inherited from `Bib::Docidentifier`.
 
 ### Serialization Round-Trip Pattern
 

data/lib/relaton/nist/docidentifier.rb

@@ -74,10 +74,6 @@ module Relaton
         original_content
       end
 
-      def to_s
-        content.to_s
-      end
-
       # `#exclude` (no args) clones the whole graph (incl. a supplement's
       # base), since `walk_chain` mutates it in place. `@raw_content` freezes
       # the stripped rendering — `content` is live from `@pubid`, and the

data/lib/relaton/nist/hit_collection.rb

@@ -356,7 +356,7 @@ module Relaton
         end
 
         index = Relaton::Index.find_or_create(
-          :nist, url: "#{GHNISTDATA}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml",
+          :nist, pages_url: PAGES_URL,
           pubid_class: ::Pubid::Nist::Identifier
         )
         # search(r) narrows candidates by number via binary search; the block

data/lib/relaton/nist/pubs_export.rb

@@ -1,3 +1,4 @@
+require "yeptris"
 require "singleton"
 
 module Relaton
@@ -61,7 +62,7 @@ module Relaton
       #
       def unzip
         Zip::File.open(DATAFILE) do |zf|
-          JSON.parse zf.first.get_input_stream.read
+          Relaton::Json.parse_external zf.first.get_input_stream.read
         end
       end
     end

data/lib/relaton/nist.rb

@@ -22,6 +22,7 @@ module Relaton
     # (for older gem versions) is rebuilt separately by the relaton-data-nist
     # crawler, not by this gem's DataFetcher.
     INDEXFILE = "index-v2"
+    PAGES_URL = "https://relaton.github.io/relaton-data-nist/"
 
     class Error < StandardError; end
 

data/lib/relaton/ogc/data_fetcher.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
+
 require "fileutils"
 require "faraday"
-require "json"
 require_relative "../ogc"
 require_relative "scraper"
 
@@ -126,7 +126,7 @@ module Relaton
         resp = Faraday.new(ENDPOINT, headers: h).get
         case resp.status
         when 200
-          json = JSON.parse(resp.body)
+          json = Relaton::Json.parse_external(resp.body)
           block_given? ? yield(resp[:etag], json) : json
         when 304 then []
         else raise Relaton::RequestError, "Could not access #{ENDPOINT}"

data/lib/relaton/omg/scraper.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "json"
+
 require "mechanize"
 require "pubid"
 
@@ -118,10 +118,10 @@ module Relaton
         script = @doc.at_xpath('//script[@type="application/ld+json"]')
         return unless script
 
-        node = JSON.parse(script.text).find { |e| e.is_a?(Hash) && e[LD_DATE] }
+        node = Relaton::Json.parse_external(script.text).find { |e| e.is_a?(Hash) && e[LD_DATE] }
         value = node && node[LD_DATE].first["@value"]
         value && ::Date.parse(value)
-      rescue JSON::ParserError, ::Date::Error
+      rescue ::JSON::ParserError, ::Date::Error
         nil
       end
 

data/lib/relaton/plateau/data_fetcher.rb

@@ -1,3 +1,4 @@
+require "yeptris"
 require "json"
 require "relaton/core"
 require_relative "../plateau"
@@ -78,7 +79,7 @@ module Relaton
         body = hadle_response(response)
 
         # Parse the JSON response
-        JSON.parse(body)
+        Relaton::Json.parse_external(body)
       rescue StandardError => e
         # Handle any errors during the fetching process
         Util.error "Error fetching JSON data from #{url}: #{e.message}"

data/lib/relaton/plateau/hit_collection.rb

@@ -30,7 +30,7 @@ module Relaton
 
       def index
         @index ||= Relaton::Index.find_or_create(
-          :plateau, url: "#{ENDPOINT}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml",
+          :plateau, pages_url: PAGES_URL,
           pubid_class: ::Pubid::Plateau::Identifier
         )
       end

data/lib/relaton/plateau.rb

@@ -19,6 +19,7 @@ require_relative "plateau/processor"
 module Relaton
   module Plateau
     INDEXFILE = "index-v2"
+    PAGES_URL = "https://relaton.github.io/relaton-data-plateau/"
 
     class Error < StandardError; end