relaton 3.0.0 → 3.0.2

data/lib/relaton/db/cache.rb

@@ -1,6 +1,6 @@
+require "yeptris"
 require "fileutils"
 require "digest"
-require "json"
 require "date"
 require "lutaml/store"
 require_relative "cache_entry"
@@ -520,7 +520,7 @@ module Relaton
       def canonical(key)
         return key if key.is_a? String
 
-        JSON.parse JSON.generate(key.to_hash)
+        Yeptris::JSON.load Yeptris::JSON.dump(key.to_hash)
       end
 
       def row_key(row)
@@ -556,7 +556,7 @@ module Relaton
 
       # A digest of the canonical key, independent of the hash order.
       def digest(key)
-        Digest::SHA256.hexdigest JSON.generate(canonical_sorted(key))
+        Digest::SHA256.hexdigest Yeptris::JSON.dump(canonical_sorted(key))
       end
 
       def canonical_sorted(key)

data/lib/relaton/db.rb

@@ -127,7 +127,15 @@ module Relaton
           rescue Relaton::RequestError => e
             args[3].call e
           rescue StandardError => e
-            Util.error "`#{args[0]}` -- #{e.message}"
+            # An unparseable reference ("ISO Standards Tag Set" in a
+            # vocabulary corpus) is a not-found input, not a failure: the
+            # citation keeps its literal text and the build continues
+            # (relaton#412). Real errors keep the ERROR level.
+            if unreadable_reference?(e)
+              Util.info "`#{args[0]}` -- #{e.message}"
+            else
+              Util.error "`#{args[0]}` -- #{e.message}"
+            end
             args[3].call nil
           end
           @queues[stdclass] =
@@ -213,6 +221,14 @@ module Relaton
 
     private
 
+    # True when the error is a flavor's grammar refusing to read the
+    # reference. Both are raised by the flavors' pubid/parslet parses; a
+    # reference they cannot read is not-found, not a failure (relaton#412).
+    def unreadable_reference?(error)
+      (defined?(Pubid::Errors::Error) && error.is_a?(Pubid::Errors::Error)) ||
+        (defined?(Parslet::ParseFailed) && error.is_a?(Parslet::ParseFailed))
+    end
+
     # The flavor routing picks for `fetch_async`'s per-flavor queue; nil
     # (logged) when no flavor recognizes the reference.
     def async_route(ref)
@@ -309,6 +325,9 @@ module Relaton
           )
         end
       end
+      # Nothing resolved: answer not-found, not a stub that holds only the
+      # reference (relaton-un#14, metanorma-ietf#205).
+      doc.relation.empty? ? nil : doc
     end
 
     # The legacy string cache key, for a processor with no pubid class. The

data/lib/relaton/doi/crossref.rb

@@ -1,3 +1,4 @@
+require "yeptris"
 require "mechanize"
 
 module Relaton
@@ -58,7 +59,7 @@ module Relaton
         url = "https://api.crossref.org/works/#{CGI.escape(id)}"
         loop do
           resp = agent.get url
-          work = JSON.parse resp.body
+          work = Relaton::Json.parse_external resp.body
           return work["message"] if work["status"] == "ok"
 
           if n > 1

data/lib/relaton/doi/parser.rb

@@ -1,3 +1,4 @@
+require "yeptris"
 module Relaton
   module Doi
     class Parser
@@ -856,7 +857,7 @@ module Relaton
         throttled = 0
         begin
           resp = Crossref.agent.get url
-          JSON.parse(resp.body).dig("message", "items")
+          Relaton::Json.parse_external(resp.body).dig("message", "items")
         rescue Mechanize::ResponseCodeError => e
           # 429 must be handled before the 4xx nil return: a throttle is not
           # "no such record". Returning nil here made a rate-limited lookup look
@@ -881,7 +882,7 @@ module Relaton
           retries += 1
           retry if retries <= MAX_RETRIES
           raise Relaton::RequestError, "Crossref network error after #{MAX_RETRIES} retries: #{e.message}"
-        rescue JSON::ParserError => e
+        rescue ::Yeptris::JSON::ParseError => e
           raise Relaton::RequestError, "Crossref JSON parsing error: #{e.message}"
         end
       end

data/lib/relaton/etsi/data_fetcher.rb

@@ -1,5 +1,5 @@
+require "yeptris"
 require "date"
-require "json"
 require "mechanize"
 require "relaton/core"
 require "relaton/index"
@@ -146,11 +146,11 @@ module Relaton
       # @raise [BadPage] when the body is not a JSON array
       #
       def parse_page(page, body)
-        records = JSON.parse(body)
+        records = Relaton::Json.parse_external(body)
         return records if records.is_a?(Array)
 
         raise BadPage, bad_page_message(page, body)
-      rescue JSON::ParserError
+      rescue ::Yeptris::JSON::ParseError
         raise BadPage, bad_page_message(page, body)
       end
 

data/lib/relaton/gb/sec_scraper.rb

@@ -1,8 +1,8 @@
 # encoding: UTF-8
 # frozen_string_literal: true
 
+
 require "net/http"
-require "json"
 require "moxml"
 require_relative "scraper"
 require_relative "item"
@@ -24,8 +24,8 @@ module Relaton
           uri = URI "https://hbba.sacinfo.org.cn/stdQueryList"
           resp = Net::HTTP.post uri, URI.encode_www_form({ key: text }),
                                 "Content-Type" => "application/x-www-form-urlencoded"
-          # res = JSON.parse Net::HTTP.get(uri)
-          json = JSON.parse resp.body
+          # res = Relaton::Json.parse_external Net::HTTP.get(uri)
+          json = Relaton::Json.parse_external resp.body
           hits = json["records"].map do |h|
             Hit.new pid: h["pk"], docref: h["code"], status: h["status"], scraper: self
           end

data/lib/relaton/iana/data_fetcher.rb

@@ -13,12 +13,14 @@ module Relaton
       end
 
       def fetch(_source = nil)
+        docs = []
         Dir["iana-registries/**/*.xml"].each do |file|
           content = File.read file, encoding: "UTF-8"
-          parse(content) if content.include? "<registry"
+          collect content, docs if content.include? "<registry"
         rescue StandardError => e
           Util.error "Error: #{e.message}. File: #{file}"
         end
+        write_docs docs
         index.save
         report_errors
       end
@@ -50,12 +52,57 @@ module Relaton
         Util.warn "Skipping index entry for `#{docnumber}` (#{file}): #{e.message}"
       end
 
-      def parse(content)
+      # A registry element with no <record> children carries no registry data
+      # of its own: IANA wraps some registries in a same-titled <hide/> page
+      # (the language-tag-extensions wrapper around its "-1" child), and
+      # republishes child registries as standalone pages whose XML is empty
+      # (rpki-signed-objects). The saved record for such an element is a
+      # slug-titled, dateless duplicate of the registry that holds the
+      # records (relaton-iana#8, #9), so it never becomes a document: when a
+      # record-bearing document of the same registry id exists, it becomes an
+      # index alias to that document's file; otherwise it is dropped.
+      RegistryDoc = Struct.new(:bib, :empty, :source)
+
+      def collect(content, docs)
         xml = Moxml.parse(content)
         registry = xml.at_xpath("/xmlns:registry", NS)
         doc = Parser.parse registry, nil, @errors
-        save_doc doc
-        registry.xpath("./xmlns:registry", NS).each { |r| save_doc Parser.parse(r, doc, @errors) }
+        docs << RegistryDoc.new(doc, recordless?(registry), content) if doc
+        registry.xpath("./xmlns:registry", NS).each do |r|
+          child = Parser.parse r, doc, @errors
+          docs << RegistryDoc.new(child, recordless?(r), content) if child
+        end
+      end
+
+      def recordless?(registry_el)
+        registry_el.xpath("./xmlns:record", NS).empty?
+      end
+
+      def write_docs(docs)
+        saved = {}
+        docs.each do |d|
+          if d.empty
+            Util.warn "Registry #{d.bib.docnumber} has no records; skipping"
+            next
+          end
+          # The fetched XML content bounds the registry's record, so an
+          # unchanged source skips the parse and the write (the digest
+          # ledger); the aliased file still has to resolve for the alias
+          # phase. The content is only digested when the ledger is on.
+          if @digest_ledger && d.source && unchanged?(d.bib.docnumber, d.source)
+            saved[d.bib.id] = ledger_file(d.bib.docnumber)
+            next
+          end
+          file = save_doc d.bib
+          note_digest d.bib.docnumber, d.source, file: file if @digest_ledger && d.source
+          saved[d.bib.id] = file if file
+        end
+        docs.select(&:empty).each do |d|
+          next unless (file = saved[d.bib.id])
+
+          Util.warn "Registry #{d.bib.docnumber} has no records; aliasing #{file}"
+          add_to_index d.bib.docnumber, file
+        end
       end
 
       #
@@ -83,6 +130,8 @@ module Relaton
         @files << file
         add_to_index bib.docnumber, file
         File.write file, serialize(bib), encoding: "UTF-8"
+        record_written file
+        file
       end
 
       def to_xml(bib)

data/lib/relaton/iec/data_fetcher.rb

@@ -1,3 +1,4 @@
+require "yeptris"
 require "relaton/core"
 require_relative "../iec"
 require_relative "data_parser"
@@ -135,7 +136,7 @@ module Relaton
             Util.warn "#{res.body}"
             break
           end
-          json = JSON.parse res.body
+          json = Relaton::Json.parse_external res.body
           json[list_key].each { |pub| fetch_pub pub }
           page += 1
           next_page = res["link"]&.include? "rel=\"last\""
@@ -191,7 +192,7 @@ module Relaton
           res = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
             http.request req
           end
-          JSON.parse(res.body)["access_token"]
+          Relaton::Json.parse_external(res.body)["access_token"]
         end
       end
 

data/lib/relaton/iec/model/docidentifier.rb

@@ -46,10 +46,6 @@ module Relaton
         original_content
       end
 
-      def to_s
-        content.to_s
-      end
-
       # `remove_part!`/`remove_date!` are unnecessary here: `to_all_parts`'s
       # own identity computation already excludes part/date (down the whole
       # base chain, so a supplement's own year is dropped too — a disclosed

data/lib/relaton/ieee/CLAUDE.md

@@ -24,10 +24,36 @@ serialized to the `_type: pubid:ieee:*` (lutaml) form, keyed on the index via
   loss is never silent. `DataFetcher.fetch` (doc generation) is unchanged — the
   data repo (`relaton-data-ieee`) calls `build_index` after obtaining `data/`.
 - **Read side.** `bibliography.rb#search` passes `pubid_class:` to `find_or_create`,
-  parses the query with `parse_pubid` (falling back to the raw String on failure),
-  passes the pubid to `index.search` for number-narrowing, and picks the row by
-  `row[:id].to_s`. `processor.rb#remove_index_file` passes `url: true` and `file:`
-  only: the delete never reads the index (see `lib/relaton/index/CLAUDE.md`).
+  parses the query with `parse_pubid` (falling back to the raw String on failure)
+  and passes the pubid to `index.search` for number-narrowing. A parsed query
+  selects its rows by **pubid equality with the year aside**
+  (`id.exclude(:year) == pubid.exclude(:year)`), so a redline, a draft, a
+  corrigendum and another number (`1003.10` for `1003.1`) never match. A dated
+  query (its own year, or the `year` argument) takes that year; an **undated**
+  query takes the **latest** edition, and `get` returns it as
+  `to_most_recent_reference` (undated IEEE docids, no date, an `instanceOf`
+  relation to the edition; `opts[:keep_year]` keeps it dated) — relaton-ieee#29.
+  Before this, a substring match + `min_by(to_s)` answered `IEEE 528` with the
+  oldest edition, `528-1994`. When equality finds nothing (an unparseable query,
+  or one of the ~4% of forms whose year pubid keeps elsewhere, e.g.
+  `IEEE Std 1003.1-1988/INT-1992`, the corrigendum's legacy base year), the old
+  substring match is the fallback. `processor.rb#remove_index_file` passes
+  `url: true` and `file:` only: the delete never reads the index (see
+  `lib/relaton/index/CLAUDE.md`).
+- **Why not `pubid === row`.** pubid declares no `subset_strict` attribute for
+  IEEE, so a nil `parts`, `prefix`, `draft` or `draft_status` in the query is a
+  wildcard: over the published index (12,616 rows) `===` let `IEEE 802.3` match
+  `802.3.1-2011` and `IEEE 754` match `IEEE Unapproved P754/D9.0`. Switch to
+  `index.search(pubid)` once pubid makes those strict, and re-measure `===`
+  against the equality over the full index first.
+- **`Relaton::Ieee::Docidentifier`.** `Item` holds this class (the producer
+  creates it too: `IdamsParser#docidentifier`, `DataFetcher`), because
+  `Bib::Docidentifier#remove_date!` is a no-op. Its `remove_date!` re-renders an
+  `IEEE` docid through pubid without the year (`to_s(trademark: true)` for the
+  `scope: trademark` one) and leaves an id **with no year** untouched: a
+  re-render would change a draft's spelling (`P802.5t™/D-2.5` →
+  `P802.5t®/D2.5`). `ItemData#to_most_recent_reference` drops the ISBN and
+  DOI, which name one edition.
 
 ## Identifier resolution (`RawbibIdParser`, pubid-first)
 

data/lib/relaton/ieee/bibliography.rb

@@ -9,33 +9,12 @@ module Relaton
         #
         # @param ref [String, Pubid::Ieee::Identifier] the reference, or its
         #   parse from Relaton::Db (relaton#205)
+        # @param year [String, nil] the year, when the reference has none
         #
         # @return [Relaton::Ieee::ItemData, nil]
         #
-        def search(ref) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-          # The machine index on the Pages site (relaton#206, the W3C
-          # pilot's contract): a parsed query reads only its own shard. A
-          # Pages failure raises `Relaton::RequestError`; the whole-index
-          # zip on the same site serves only the retry on a miss.
-          index = Relaton::Index.find_or_create :ieee, pages_url: PAGES_URL,
-                                                       pubid_class: ::Pubid::Ieee::Identifier
-          # Pass the parsed pubid (not the raw String) so index-v2 narrows
-          # candidates by number via binary search before the block runs; the
-          # block keeps the broad substring match the string index gave, and an
-          # unparseable/partial ref falls back to the full-scan String search.
-          # Rows are Pubid::Ieee::Identifier objects (not Comparable), so pick by
-          # the string form.
-          pubid = parse_pubid ref
-          needle = pubid.to_s
-          # pubid's normalization moves a legacy base year into its own
-          # comma segment ("IEEE P802.16/D5, 2004/Cor. 1-2005"); the
-          # index rows carry the canonical form without it, so the
-          # block also matches with the moved year stripped
-          row = index.search(pubid) do |r|
-            id = r[:id].to_s
-            id.include?(needle) ||
-              id.include?(needle.gsub(/, (19|20)\d{2}\//, "/"))
-          end.min_by { |r| r[:id].to_s }
+        def search(ref, year = nil)
+          row = find_row parse_pubid(ref), year
           return unless row
 
           resp = Faraday.get "#{GH_URL}#{row[:file]}"
@@ -47,30 +26,106 @@ module Relaton
         end
 
         #
-        # Get IEEE bibliography item by reference.
+        # Get IEEE bibliography item by reference. An undated reference
+        # answers with the latest edition as an undated item that has an
+        # `instanceOf` relation to that edition (relaton-ieee#29).
         #
         # @param ref [String, Pubid::Ieee::Identifier] the IEEE standard
         #   reference to look up (e.g. "IEEE 528-2019"), or its parse from
         #   Relaton::Db (relaton#205)
         # @param year [String] the year the standard was published (optional)
         # @param opts [Hash] options
+        # @option opts [Boolean] :keep_year keep the edition's date on an
+        #   undated reference
         #
         # @return [Relaton::Ieee::ItemData, nil]
         #
-        def get(ref, _year = nil, _opts = {}) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        def get(ref, year = nil, opts = {}) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
           Util.info "Fetching from Relaton repository ...", key: ref.to_s
-          item = search(ref)
-          if item
-            Util.info "Found: `#{item.docidentifier.first.content}`", key: ref.to_s
-            item
-          else
+          item = search(ref, year)
+          unless item
             Util.info "Not found.", key: ref.to_s
-            nil
+            return
           end
+
+          Util.info "Found: `#{item.docidentifier.first.content}`", key: ref.to_s
+          return item if opts[:keep_year] || dated?(parse_pubid(ref), year)
+
+          item.to_most_recent_reference
         end
 
         private
 
+        # The index row of the reference. A parsed reference matches the rows
+        # of its own document by pubid equality, the year aside; an undated
+        # one takes the latest edition. When that finds nothing (an
+        # unparseable reference, or a form whose year pubid keeps elsewhere)
+        # the broad substring match is the fallback.
+        #
+        # @param pubid [Pubid::Ieee::Identifier, String]
+        # @param year [String, nil]
+        # @return [Hash, nil] the `{ id:, file: }` row
+        def find_row(pubid, year)
+          index = Relaton::Index.find_or_create :ieee, pages_url: PAGES_URL,
+                                                       pubid_class: ::Pubid::Ieee::Identifier
+          row = pubid_row(index, pubid, year) unless pubid.is_a?(String)
+          row || substring_row(index, pubid)
+        end
+
+        # Not `pubid === row`: IEEE `parts`, `prefix`, `draft` and
+        # `draft_status` are not `subset_strict` in pubid, so a nil there is a
+        # wildcard and `IEEE 802.3` would match `IEEE Std 802.3.1-2011` and
+        # `IEEE Unapproved P802.3/D2.0`.
+        def pubid_row(index, pubid, year)
+          target = (pubid.year || year)&.to_s
+          base = pubid.exclude(:year)
+          rows = index.search(pubid) do |r|
+            id = r[:id]
+            id.is_a?(::Pubid::Ieee::Identifier) && id.exclude(:year) == base &&
+              (target.nil? || id.year.to_s == target)
+          end
+          return rows.min_by { |r| r[:id].to_s } if target
+
+          rows.max_by { |r| [r[:id].year.to_s, r[:id].to_s] }
+        end
+
+        # pubid's normalization moves a legacy base year into its own comma
+        # segment ("IEEE P802.16/D5, 2004/Cor. 1-2005"); the index rows carry
+        # the canonical form without it, so the block also matches with the
+        # moved year stripped. Rows are Pubid::Ieee::Identifier objects (not
+        # Comparable), so pick by the string form.
+        #
+        # The project prefix moves too: "IEEE P802.21/D14" parses as a
+        # ProjectDraft, but the pre-publication crawl records spell the same
+        # document "IEEE Unapproved 802.21/D14, Sept 2008" — the Unapproved
+        # draft status occupies the slot the publisher and P would render in,
+        # so the row string contains neither (relaton-ieee#25). The draft
+        # number "/D14" still has to match, so the bare form cannot reach the
+        # published standard or a different draft — the variant exists only
+        # for a reference that names its draft.
+        def substring_row(index, pubid)
+          needle = pubid.to_s
+          # The publisher-and-prefix-stripped form; nil when there is no
+          # project-prefixed draft number.
+          stripped = needle.sub("IEEE P", "") if needle.match?(/\AIEEE P\S*\/D\d/i)
+          index.search(pubid) do |r|
+            id = r[:id].to_s
+            id.include?(needle) ||
+              id.include?(needle.gsub(/, (19|20)\d{2}\//, "/")) ||
+              (stripped && id.include?(stripped))
+          end.min_by { |r| r[:id].to_s }
+        end
+
+        # @param pubid [Pubid::Ieee::Identifier, String]
+        # @param year [String, nil]
+        def dated?(pubid, year)
+          return true if year
+          return pubid.year if pubid.respond_to?(:year)
+
+          # An unparseable reference was found by substring; leave it dated.
+          true
+        end
+
         # Parse a reference into a Pubid::Ieee::Identifier for index narrowing, or
         # return the raw String when pubid can't parse it (e.g. a partial ref) so
         # the search falls back to the substring scan.

data/lib/relaton/ieee/data_fetcher.rb

@@ -171,7 +171,7 @@ module Relaton
         end
         return if RELATION_TYPES[type] == false
 
-        docid = Bib::Docidentifier.new(type: "IEEE", content: fref, primary: true)
+        docid = Docidentifier.new(type: "IEEE", content: fref, primary: true)
         bib = ItemData.new formattedref: Bib::Formattedref.new(content: fref), docidentifier: [docid]
         description = create_relation_description type
         Bib::Relation.new(type: RELATION_TYPES[type][:type], description: description, bibitem: bib)

data/lib/relaton/ieee/docidentifier.rb

@@ -0,0 +1,37 @@
+module Relaton
+  module Ieee
+    # IEEE document identifier. An `IEEE` docid is backed by a
+    # Pubid::Ieee::Identifier so `remove_date!` can strip the year when an
+    # undated reference answers with its latest edition (relaton-ieee#29).
+    # ISBN and DOI docids carry no pubid and are never changed.
+    class Docidentifier < Bib::Docidentifier
+      # Strip the year: `IEEE Std 528-2019` -> `IEEE Std 528`, and the
+      # trademark docid `IEEE Std 528™-2019` -> `IEEE Std 528™`. An id with no
+      # year is left as it is: a re-render would change its spelling
+      # (`IEEE P802.5t™/D-2.5` -> `IEEE P802.5t®/D2.5`).
+      def remove_date!
+        pubid = parse_pubid
+        return unless pubid&.year
+
+        undated = pubid.exclude(:year)
+        self.content = scope == "trademark" ? undated.to_s(trademark: true) : undated.to_s
+      end
+
+      # IEEE has no all-parts form: both are no-ops, so
+      # `Bib::ItemData#to_all_parts` never raises.
+      def remove_part!; end
+
+      def to_all_parts!; end
+
+      private
+
+      def parse_pubid
+        return unless type == "IEEE" && content
+
+        ::Pubid::Ieee::Identifier.parse content
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/relaton/ieee/idams_parser.rb

@@ -92,7 +92,7 @@ module Relaton
       #
       # Parse identifiers
       #
-      # @return [Array<Relaton::Bib::Docidentifier>]
+      # @return [Array<Relaton::Ieee::Docidentifier>]
       #
       def parse_docidentifier # rubocop:disable Metrics/MethodLength
         ids = @doc.isbn_doi.map { |id| id[:content] = id.delete(:id); id }
@@ -102,7 +102,7 @@ module Relaton
           ids.unshift(content: pubid.to_s, type: "IEEE", primary: true)
         end
 
-        result = ids.map { |dcid| Bib::Docidentifier.new(**dcid) }
+        result = ids.map { |dcid| Docidentifier.new(**dcid) }
         @errors[:docidentifier] &&= result.empty?
         result
       end

data/lib/relaton/ieee/item.rb

@@ -1,4 +1,5 @@
 require_relative "ext"
+require_relative "docidentifier"
 
 module Relaton
   module Ieee
@@ -6,6 +7,8 @@ module Relaton
       model ItemData
 
       attribute :ext, Ext
+      attribute :docidentifier, Docidentifier, collection: true,
+                                               initialize_empty: true
     end
   end
 end

data/lib/relaton/ieee/item_data.rb

@@ -1,7 +1,13 @@
 module Relaton
   module Ieee
     class ItemData < Bib::ItemData
-
+      # The ISBN and DOI name one edition, so the undated item keeps only the
+      # IEEE docids; its `instanceOf` relation still carries them.
+      def to_most_recent_reference
+        me = super
+        me.docidentifier = me.docidentifier.select { |d| d.type == "IEEE" }
+        me
+      end
     end
   end
 end

data/lib/relaton/ietf/CLAUDE.md

@@ -36,7 +36,7 @@ Converter classes use `Bib::NamespaceHelper` which resolves `namespace` by takin
 
 ### Key Flows
 
-1. **Single document lookup**: `Processor#get` → `Bibliography.get(code)` → `Scraper.scrape_page` → parses the reference with `Pubid::Ietf` → searches the one combined `index-v2` from `relaton-data-ietf` → fetches that row's YAML
+1. **Single document lookup**: `Processor#get` → `Bibliography.get(code)` → `Scraper.scrape_page` → parses the reference with `Pubid::Ietf` → searches the one combined `index-v2` from `relaton-data-ietf` → fetches that row's YAML; a versioned draft the index lacks is read from bib.ietf.org (see below)
 
 2. **Bulk data fetching**: `DataFetcher` extends `Relaton::Core::DataFetcher` with three datasets:
    - `ietf-rfcsubseries` / `ietf-rfc-entries`: parse `rfc-index.xml` via `Rfc::Index` / `Rfc::Entry#to_item`
@@ -119,6 +119,30 @@ The old plain-string index matched that by substring; matching is exact now.
 A reference pubid cannot parse returns nil, so `Bibliography.get "CN 8341"` logs
 "Not found." rather than raising.
 
+### An index miss on a versioned draft reads bib.ietf.org (relaton-ietf#125)
+
+The data repo crawls the rsync `bibxml-ids` snapshot, and that snapshot keeps
+only some versions of each draft: `draft-ietf-vcarddav-vcardrev` has only `-22`,
+so `-06` was "Not found." When `fetch_doc` misses and the query is a
+`Pubid::Ietf::Identifiers::InternetDraft` **with a version**, `fetch_bibxml` reads
+`https://bib.ietf.org/public/rfc/bibxml-ids/reference.I-D.<draft>-NN.xml`
+(`BIBXML_IDS`). That service resolves any version the datatracker knows and
+answers 404 otherwise; the datatracker's own `doc/bibxml3/...` URL answers 404
+for these old versions. The parse is the crawler's: `BibXMLParser.to_utf8`
+(the CP1252 repair, shared with `DataFetcher#read_bibxml`), `BibXMLParser.parse`,
+and the version from the identifier, as `process_singleton` sets it.
+
+Only a 404 is "not found". Any other non-200 (429, 5xx) raises
+`Relaton::RequestError`, so `Db#net_retry` retries it and `Db::Cache` does not
+store an outage as a not-found row. A 200 whose body does not parse logs a
+warning and returns nil; that rescue names the parse errors and is not
+`StandardError`, so a transport error still becomes `RequestError`.
+
+RFCs, sub-series and unversioned drafts do **not** fall back: the index holds
+all of them, so a miss is a real "not found". A record read this way has no
+`updates`/`updatedBy` neighbours — the crawl links those, and one fetch cannot.
+Old bibxml carries no `target`, so such a record has a `TXT` URI and no `src`.
+
 ### The producer: `DataFetcher` writes `index-v2`
 
 `DataFetcher` builds the index during the crawl, from the identifier it already
@@ -294,7 +318,8 @@ smart quotes and accented Latin letters. `File.read(encoding: "UTF-8")` only
 *tags* a string; it validates nothing, so those reached lutaml as invalid UTF-8
 and `Rfcxml::V3::Reference.from_xml` raised `Lutaml::Model::InvalidFormatError`.
 
-`read_bibxml` transcodes them (`data_fetcher.rb`). Three things make that the
+`read_bibxml` transcodes them through `BibXMLParser.to_utf8`
+(`bibxml_parser.rb`, shared with the scraper's bib.ietf.org fallback). Three things make that the
 right call rather than `scrub`:
 
 - **All of them decode losslessly as CP1252** — 125 of the ~122k published files

data/lib/relaton/ietf/bibxml_parser.rb

@@ -13,6 +13,44 @@ module Relaton
         FromRfc.new(rfc).transform
       end
 
+      #
+      # Decode bibxml bytes as UTF-8, recovering Windows-1252 bytes.
+      #
+      # These files declare `encoding='UTF-8'` but some carry CP1252 — smart
+      # quotes and accented Latin letters. `File.read(encoding: "UTF-8")` only
+      # *tags* the string, so those reach lutaml as invalid UTF-8 and it raises.
+      #
+      # `scrub` with a block transcodes each invalid *run* as CP1252 while
+      # leaving valid UTF-8 untouched. Both halves matter:
+      #
+      # * Not plain `scrub`, which substitutes U+FFFD: `client’s` would become
+      #   `client�s` and `Muñoz` `Mu�oz` — author surnames included.
+      #   The damage is length-preserving, so a length check will not catch it.
+      # * Not a whole-file CP1252 re-decode, which mangles a file that is
+      #   genuinely UTF-8 apart from one stray byte: `Muñoz café ’` would come
+      #   back as `Muñoz café ’`, silently, since the result is valid UTF-8 and
+      #   nothing raises. No such file exists in today's corpus (0 of the 125
+      #   affected contain valid multi-byte UTF-8) but it grows daily, and
+      #   "decodes losslessly as CP1252" is weak evidence of correctness —
+      #   CP1252 maps 251 of 256 byte values.
+      #
+      # `undef: :replace` covers the five bytes CP1252 leaves undefined
+      # (0x81 0x8D 0x8F 0x90 0x9D), so this returns valid UTF-8 rather than
+      # raising and costing the whole file.
+      #
+      # @param bytes [String] raw bytes, any encoding tag
+      # @return [String] UTF-8, valid encoding
+      #
+      def to_utf8(bytes)
+        utf8 = bytes.dup.force_encoding(Encoding::UTF_8)
+        return utf8 if utf8.valid_encoding?
+
+        utf8.scrub do |bad|
+          bad.force_encoding(Encoding::WINDOWS_1252)
+             .encode(Encoding::UTF_8, undef: :replace)
+        end
+      end
+
       def pubid_type(id)
         pref = id.match(/^(\S+)/)[1]
         case pref