relaton 3.0.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/ietf/data_fetcher.rb

@@ -1,5 +1,7 @@
 require "etc"
 require "parallel"
+require "pubid"
+require "pubid/ietf"
 require "relaton/core"
 require_relative "../ietf"
 require_relative "bibxml_parser"
@@ -21,12 +23,30 @@ module Relaton
         when "ietf-rfc-entries" then fetch_ieft_rfcs
         end
         index.save
+        report_unindexed
+        report_unparsed
+        report_collisions
       end
 
       private
 
+      # The published index is the pubid-structured `index-v2` (relaton#109).
+      # `pubid_class:` is not decoration: `FileIO#save` serialises an id to its
+      # `_type:` hash only when it is an instance of the configured class, so
+      # without it — or without parsing the id below — this writes a v1-shaped
+      # file under a v2 name.
+      # `url: nil` is load-bearing, not decoration. Scraper opens the same
+      # `:IETF` pool key with a `url:`, and `Type#actual?` skips the URL check
+      # when the caller omits it (`!args.key?(:url)`) — so in a process where a
+      # lookup ran first, omitting it here would hand the crawl the
+      # remote-backed Type and `save` would write to `~/.relaton/ietf/` instead
+      # of `./`, publishing no index at all. Passing it explicitly forces a
+      # local-file Type.
       def index
-        @index ||= Relaton::Index.find_or_create :IETF, file: "#{INDEXFILE}.yaml"
+        @index ||= Relaton::Index.find_or_create(
+          :IETF, url: nil, file: "#{INDEXFILE}.yaml",
+                 pubid_class: ::Pubid::Ietf::Identifier
+        )
       end
 
       #
@@ -34,8 +54,16 @@ module Relaton
       #
       def fetch_ieft_rfcsubseries
         idx = Rfc::Index.from_xml(rfc_index)
+        # Keyed by the normalised doc-id, built once for the whole crawl:
+        # `Entry` looks constituents up by `Entry.squish(ref)`, and doing it
+        # per-entry over ~9,800 RFCs would rebuild this table 367 times.
         rfc_map = (idx.rfc_entries || []).each_with_object({}) do |entry, h|
-          h[entry.doc_id] = entry
+          key = Rfc::Entry.squish(entry.doc_id)
+          if h.key?(key)
+            Util.warn "Duplicate RFC doc-id `#{entry.doc_id}` after normalisation " \
+                      "(`#{key}`); the later entry wins for constituent lookup"
+          end
+          h[key] = entry
         end
         idx.subseries_entries.each do |entry|
           save_doc entry.to_item(rfc_map, wg_names: wg_names)
@@ -54,6 +82,10 @@ module Relaton
       #
       def fetch_ieft_internet_drafts
         series_groups, singleton_paths = group_draft_paths
+        # Workers fork from here, so `unique_output_file`'s reservation cannot
+        # see a peer's claim and `write_unique` must never overwrite. Set
+        # before the first fork, so the children inherit it.
+        @cross_process = true
 
         series_results = parallelize(series_groups.to_a) do |(series, paths_info)|
           process_series(series, paths_info)
@@ -63,7 +95,13 @@ module Relaton
           process_singleton(path)
         end
 
-        (series_results + singleton_results).compact.each { |r| record_index_entry(r) }
+        entries, unparsed = (series_results + singleton_results).compact
+          .partition { |r| r[:unparsed].nil? }
+        # Tallied here, not in the worker: a counter incremented in a Parallel
+        # worker process is lost on the way back (see record_index_entry).
+        @unparsed = unparsed.map { |r| "#{r[:unparsed]} (#{r[:error]})" }
+        reconcile_output_files entries
+        entries.each { |r| record_index_entry(r) }
       end
 
       #
@@ -114,16 +152,22 @@ module Relaton
       # for the parent.
       #
       def process_series(series, paths_info)
-        sorted = paths_info.sort_by { |p| p[:ver].to_i }.map do |p|
-          bib = BibXMLParser.parse(File.read(p[:path], encoding: "UTF-8"))
+        parsed = paths_info.sort_by { |p| p[:ver].to_i }.map do |p|
+          bib, marker = parse_bibxml(p[:path])
+          next marker unless bib
+
           bib.version = [Bib::Version.new(draft: p[:ver])]
           p.merge(bib: bib, source: bib.source)
         end
+        # A file that failed to parse must not reach `sorted`:
+        # link_neighbor_relations and build_unversioned_doc both dereference
+        # `entry[:bib]`, and a dropped version is better than a nil one.
+        sorted, skipped = parsed.partition { |e| e[:unparsed].nil? }
         link_neighbor_relations(sorted) if @format != "bibxml"
 
         results = sorted.map { |entry| serialize_and_write(entry[:bib]) }
         results << serialize_and_write(build_unversioned_doc(series, sorted)) if @format != "bibxml"
-        results.compact
+        results.compact + skipped
       end
 
       #
@@ -133,11 +177,82 @@ module Relaton
         file = File.basename(path, ".xml")
         is_draft = file.include?("D.draft-")
         ver = is_draft ? file[/(\d+)$/, 1] : nil
-        bib = BibXMLParser.parse(File.read(path, encoding: "UTF-8"))
+        bib, marker = parse_bibxml(path)
+        return marker unless bib
+
         bib.version = [Bib::Version.new(draft: ver)] if ver
         serialize_and_write(bib)
       end
 
+      #
+      # Read and parse one bibxml file, or nil if it cannot be parsed.
+      #
+      # Rescues `StandardError` rather than a narrow list on purpose: lutaml
+      # raises `InvalidFormatError` on bad bytes, but the converter also runs
+      # regexes over parsed text (`parse_surname_initials` and friends), and
+      # those raise `ArgumentError: invalid byte sequence` on anything that slips
+      # through. One unparseable file must cost one document, never the crawl —
+      # the drafts path runs under Parallel.map, which discards every result from
+      # the pass when a worker raises.
+      #
+      # @param path [String]
+      # @return [Relaton::Ietf::ItemData, nil]
+      #
+      # @param path [String]
+      # @return [Array(Relaton::Ietf::ItemData, nil), Array(nil, Hash)]
+      #   the record, or nil plus a marker carrying why
+      def parse_bibxml(path)
+        bib = BibXMLParser.parse(read_bibxml(path))
+        bib ? [bib, nil] : [nil, unparsed_marker(path, "parser returned no record")]
+      rescue StandardError => e
+        [nil, unparsed_marker(path, "#{e.class}: #{e.message.to_s.lines.first.to_s.strip}")]
+      end
+
+      # Marshal-friendly stand-in for a record, carried back to the parent so the
+      # skip can be counted where a tally survives. The reason rides along rather
+      # than sitting in an ivar, which a later file would overwrite.
+      def unparsed_marker(path, error)
+        { unparsed: path, error: error }
+      end
+
+      #
+      # Read a bibxml file as UTF-8, recovering Windows-1252 bytes.
+      #
+      # These files declare `encoding='UTF-8'` but some carry CP1252 — smart
+      # quotes and accented Latin letters. `File.read(encoding: "UTF-8")` only
+      # *tags* the string, so those reach lutaml as invalid UTF-8 and it raises.
+      #
+      # `scrub` with a block transcodes each invalid *run* as CP1252 while
+      # leaving valid UTF-8 untouched. Both halves matter:
+      #
+      # * Not plain `scrub`, which substitutes U+FFFD: `client’s` would become
+      #   `client\uFFFDs` and `Muñoz` `Mu\uFFFDoz` — author surnames included.
+      #   The damage is length-preserving, so a length check will not catch it.
+      # * Not a whole-file CP1252 re-decode, which mangles a file that is
+      #   genuinely UTF-8 apart from one stray byte: `Muñoz café ’` would come
+      #   back as `Muñoz café ’`, silently, since the result is valid UTF-8 and
+      #   nothing raises. No such file exists in today's corpus (0 of the 125
+      #   affected contain valid multi-byte UTF-8) but it grows daily, and
+      #   "decodes losslessly as CP1252" is weak evidence of correctness —
+      #   CP1252 maps 251 of 256 byte values.
+      #
+      # `undef: :replace` covers the five bytes CP1252 leaves undefined
+      # (0x81 0x8D 0x8F 0x90 0x9D), so this returns valid UTF-8 rather than
+      # raising and costing the whole file.
+      #
+      # @param path [String]
+      # @return [String] UTF-8, valid encoding
+      #
+      def read_bibxml(path)
+        utf8 = File.binread(path).force_encoding(Encoding::UTF_8)
+        return utf8 if utf8.valid_encoding?
+
+        utf8.scrub do |bad|
+          bad.force_encoding(Encoding::WINDOWS_1252)
+             .encode(Encoding::UTF_8, undef: :replace)
+        end
+      end
+
       #
       # Append immediate-neighbor `updates` / `updatedBy` relations in memory.
       # Single-version series get no relations (no neighbors).
@@ -160,6 +275,12 @@ module Relaton
       # `includes` relations to every version. Uses the latest version's
       # title/abstract from memory.
       #
+      # The aggregator is *synthesised* — there is no upstream document for it,
+      # so `date`, `ext` (hence doctype) and `source` can only be inherited from
+      # its newest constituent, which `sorted` already holds in memory. Without
+      # that inheritance it publishes undated (and so unsorted on the Pages
+      # index, which sorts by date) and with no document type at all.
+      #
       # @return [Relaton::Ietf::ItemData, nil]
       #
       def build_unversioned_doc(series, sorted)
@@ -173,7 +294,11 @@ module Relaton
         rel = sorted.map { |e| version_relation({ ref: e[:ref], source: e[:source] }, "includes") }
         ItemData.new(
           title: last_v.title, abstract: last_v.abstract, formattedref: Bib::Formattedref.new(content: series),
-          docidentifier: [docid], relation: rel
+          docidentifier: [docid], relation: rel,
+          # dup'd, not shared: these are the newest version's own objects, and
+          # aliasing them would make any later edit to the aggregator mutate the
+          # `-NN` record too.
+          date: last_v.date&.dup, ext: last_v.ext&.dup, source: last_v.source&.dup
         )
       end
 
@@ -254,10 +379,37 @@ module Relaton
                entry.docidentifier.detect { |i| i.type == "Internet-Draft" && i.primary }&.content
              end
         id ||= entry.docnumber || entry.formattedref.content
-        file = output_file(id)
-        File.write file, content, encoding: "UTF-8"
+        file = write_unique(id, content)
         primary = entry.docidentifier.detect(&:primary) || entry.docidentifier.first
-        { docnumber: entry.docnumber, file: file, index_id: primary.content }
+        # `docid` is the id the file was written under and `plain_file` the name
+        # it would have taken uncontested; `reconcile_output_files` needs both.
+        # Neither is `index_id`: `id` falls back through docnumber and
+        # formattedref, so on the RFC path the two differ ("RFC0001" vs "RFC 1").
+        { docnumber: entry.docnumber, docid: id, file: file,
+          plain_file: output_file(id), index_id: primary.content,
+          pubid: parse_pubid(primary.content) }
+      end
+
+      #
+      # Parse a record's primary docidentifier into the pubid the index stores.
+      #
+      # Deliberately here rather than in `record_index_entry`: this runs inside
+      # the `Parallel` workers, and a pubid identifier survives the Marshal round
+      # trip Parallel does on the return value. Parsing in the parent instead
+      # would put ~0.7 ms per record back on the serial path — some minutes over
+      # the 167k-draft crawl, all of it outside the parallelism this fetcher is
+      # built around.
+      #
+      # @param [String] content primary docidentifier content
+      # @return [Pubid::Ietf::Identifier, nil] nil when pubid rejects it
+      #
+      def parse_pubid(content)
+        ::Pubid::Ietf::Identifier.parse content
+      rescue StandardError => e
+        # Full message: the tail is the part that says *what shape* pubid
+        # stopped accepting, which is the whole point of the warning.
+        Util.warn "Not indexing `#{content}`: #{e.message}"
+        nil
       end
 
       #
@@ -270,7 +422,138 @@ module Relaton
         elsif check_duplicate
           @files << result[:file]
         end
-        index.add_or_update result[:index_id], result[:file]
+        # A record whose identifier pubid rejects is written but not indexed —
+        # never fatal. The index load is all-or-nothing (`deserialize_id` raises
+        # on the first bad id and `load_index` then rejects the *entire* index),
+        # so one malformed upstream record must cost one document, not every
+        # lookup. All 176,862 published ids parse today; this guards drift.
+        # Counted here, in the parent, because a worker's tally would be lost.
+        unless result[:pubid]
+          @unindexed = @unindexed.to_i + 1
+          return
+        end
+
+        index.add_or_update result[:pubid], result[:file]
+      end
+
+      #
+      # Settle, in the parent, which record keeps which filename.
+      #
+      # `output_file` is not injective, so distinct docids can want one path.
+      # `write_unique` refuses to clobber, but a forked worker cannot know
+      # WHICH of the clashing docids deserves the plain name — it only knows the
+      # path was taken. Left there, the winner would follow the race and the two
+      # filenames would swap between crawls, churning the data repo.
+      #
+      # So the parent decides once it can see every docid: within a group of
+      # records that wanted one path, the alphabetically first docid keeps it
+      # and the rest take their digest variant. Runs over EVERY group, not just
+      # clashing ones — a lone record that fell back to a digest path (a
+      # leftover file, any transient clash) must get its plain name back, or the
+      # published filename churns and the old file is orphaned.
+      #
+      # @param [Array<Hash>] results worker results, mutated in place so
+      #   `record_index_entry` indexes the final path
+      #
+      def reconcile_output_files(results)
+        results.group_by { |r| r[:plain_file] }.each do |plain, group|
+          next if plain.nil? # hand-built results, and the bibxml format
+
+          targets = assign_output_files plain, group
+          record_collision targets
+          filled = Set.new
+          order(group, targets, plain).each do |result|
+            target = targets[result[:docid].to_s]
+            place_output_file result, target, filled.add?(target).nil?
+          end
+        end
+      end
+
+      # docid => the filename it should end up with. Sorting the *unique* docids
+      # leaves no tie to break, so the assignment cannot drift between crawls
+      # the way an unstable `sort_by` over the results would.
+      def assign_output_files(plain, group)
+        group.map { |r| r[:docid].to_s }.uniq.sort.each_with_index.to_h do |docid, i|
+          [docid, i.zero? ? plain : digest_output_file(docid)]
+        end
+      end
+
+      # Records already at their target first (they are no-ops), then the movers
+      # bound for a digest path, then the one bound for `plain`. That ordering is
+      # load-bearing: a loser may be sitting ON the plain path, and moving the
+      # winner there first would destroy it.
+      def order(group, targets, plain)
+        group.sort_by do |r|
+          target = targets[r[:docid].to_s]
+          [r[:file] == target ? 0 : 1, target == plain ? 1 : 0, r[:file].to_s]
+        end
+      end
+
+      #
+      # Move one record's file to the name the parent chose for it.
+      #
+      # `duplicate` means another result for the SAME docid already holds the
+      # target. Today that is one file and one warning, so drop the stray rather
+      # than publish the document twice under two names.
+      #
+      def place_output_file(result, target, duplicate)
+        file = result[:file]
+        return result[:file] = target if file.nil? || file == target
+
+        if duplicate
+          # The document is at the target whatever happens next, so the result
+          # follows it first. The stray may ALREADY be gone: two workers that
+          # both lost the race to the plain path land on one fallback name,
+          # because that name is keyed on the docid -- so the first of them
+          # renamed this very file onto the target. Leaving the result on its
+          # old name would put an index row on a path that no longer exists.
+          result[:file] = target
+          return unless File.exist?(file)
+
+          Util.warn "Duplicate document `#{result[:docid]}`: dropping #{file}, keeping #{target}"
+          return File.delete(file)
+        end
+        return unless File.exist?(file)
+
+        File.rename file, target
+        result[:file] = target
+      rescue SystemCallError => e
+        # A late failure must not throw away a multi-hour crawl. The record keeps
+        # the name it already has on disk.
+        Util.warn "Could not move #{file} to #{target}: #{e.message}"
+      end
+
+      def record_collision(targets)
+        return if targets.size < 2
+
+        # targets.keys is already the uniqued, sorted docid list.
+        (@collisions ||= []) << targets.keys
+      end
+
+      # One line for a crawl that writes ~177k records, not one per collision.
+      def report_collisions
+        return if @collisions.nil? || @collisions.empty?
+
+        Util.warn "#{@collisions.size} filename collision(s): distinct docids sanitize to " \
+                  "one filename and were given separate files. " \
+                  "First: #{@collisions.first(5).map { |g| g.join(' <-> ') }.join('; ')}"
+      end
+
+      # Skips are per-record warnings in a crawl that writes ~177k of them, so
+      # restate the total where it can actually be noticed.
+      # One line for a crawl that reads ~167k files, not one per skip.
+      def report_unparsed
+        return if @unparsed.nil? || @unparsed.empty?
+
+        Util.warn "#{@unparsed.size} file(s) skipped: could not be parsed. " \
+                  "First: #{@unparsed.first(5).join(', ')}"
+      end
+
+      def report_unindexed
+        return unless @unindexed.to_i.positive?
+
+        Util.warn "#{@unindexed} document(s) written but not indexed: " \
+                  "identifier not parseable by Pubid::Ietf"
       end
 
     end

data/lib/relaton/ietf/processor.rb

@@ -59,9 +59,13 @@ module Relaton
       #
       def remove_index_file
         require_relative "../ietf"
-        Relaton::Index.find_or_create(:RFC, url: true, file: "#{INDEXFILE}.yaml").remove_file
-        Relaton::Index.find_or_create(:RSS, url: true, file: "#{INDEXFILE}.yaml").remove_file
-        Relaton::Index.find_or_create(:IDS, url: true, file: "#{INDEXFILE}.yaml").remove_file
+        Relaton::Index.find_or_create(:IETF, url: true, file: "#{INDEXFILE}.yaml").remove_file
+        # Also clear the three per-type caches a previously released relaton
+        # left in ~/.relaton; nothing writes them now, so without this they
+        # would sit there forever, unreachable by `relaton clear`.
+        %i[RFC RSS IDS].each do |type|
+          Relaton::Index.find_or_create(type, url: true, file: "index-v1.yaml").remove_file
+        end
       end
     end
   end

data/lib/relaton/ietf/rfc/entry.rb

@@ -125,10 +125,29 @@ module Relaton
           is_also&.doc_id&.any? || false
         end
 
+        #
+        # Normalise a doc-id for constituent lookup: fold case and strip
+        # whitespace and dots.
+        #
+        # Relation targets and the docids they reference disagree on both across
+        # the published corpus (`dyndNS` vs `dyndns`), which leaves records
+        # undated — and, in `build_relations`, silently downgrades a full
+        # constituent bibitem to a minimal one — under a strict lookup. Both
+        # sides go through this: the caller keys its index by it, `Entry` looks
+        # up by it.
+        #
+        # @param id [String, nil]
+        # @return [String]
+        #
+        def self.squish(id)
+          id.to_s.gsub(/[\s.]/, "").downcase
+        end
+
         #
         # Convert to Relaton::Ietf::ItemData
         #
-        # @param rfc_index [Hash{String => Entry}, nil] lookup of RFC entries by doc-id
+        # @param rfc_index [Hash{String => Entry}, nil] lookup of RFC entries,
+        #   keyed by `Entry.squish(doc_id)` — see that method for why
         # @return [Relaton::Ietf::ItemData, nil]
         #
         def to_item(rfc_index = nil, wg_names: {})
@@ -174,6 +193,7 @@ module Relaton
             script: ["Latn"],
             source: build_link,
             formattedref: build_formattedref,
+            date: build_subseries_date(rfc_index),
             relation: build_relations(rfc_index, wg_names: wg_names),
             series: build_series,
             ext: Ext.new(doctype: Doctype.new(content: "rfc"), stream: stream, flavor: "ietf"),
@@ -211,12 +231,28 @@ module Relaton
           return [] unless is_also&.doc_id
 
           is_also.doc_id.map do |ref|
-            rfc_entry = rfc_index&.[](ref)
+            rfc_entry = rfc_index&.[](Entry.squish(ref))
             bibitem = rfc_entry ? rfc_entry.to_rfc_item(wg_names: wg_names) : build_minimal_bibitem(ref)
             Relaton::Ietf::Relation.new(type: "includes", bibitem: bibitem)
           end.compact
         end
 
+        # A sub-series entry in rfc-index.xml is a pointer and nothing more —
+        # `<doc-id>` plus `<is-also>`, with no date, title, author or status,
+        # because a sub-series has no metadata of its own. Rather than publish a
+        # dateless record, take the date of the newest RFC it includes.
+        #
+        # @param rfc_index [Hash{String => Entry}, nil]
+        # @return [Array<Bib::Date>] empty when no constituent resolves or none
+        #   carries a date
+        def build_subseries_date(rfc_index)
+          newest = (is_also&.doc_id || [])
+            .filter_map { |ref| rfc_index&.[](Entry.squish(ref)) }
+            .flat_map(&:build_rfc_date)
+            .max_by { |d| d.at.to_s }
+          newest ? [newest] : []
+        end
+
         def build_minimal_bibitem(ref)
           id = ref.sub(/^([A-Z]+)0*(\d+)$/, '\1 \2')
           docid = Bib::Docidentifier.new(type: "IETF", content: id, primary: true)
@@ -246,7 +282,7 @@ module Relaton
           [Bib::Uri.new(type: "src", content: "https://www.rfc-editor.org/info/rfc#{shortnum}")]
         end
 
-        def build_rfc_date
+        public def build_rfc_date
           (date || []).map do |d|
             month_num = ::Date::MONTHNAMES.index(d.month).to_s.rjust(2, "0")
             date_str = "#{d.year}-#{month_num}"

data/lib/relaton/ietf/scraper.rb

@@ -1,62 +1,95 @@
 # frozen_string_literal: true
 
+require "pubid"
+require "pubid/ietf"
+
 module Relaton
   module Ietf
     # Scraper module
     module Scraper
       extend Scraper
 
-      IDS = "https://raw.githubusercontent.com/relaton/relaton-data-ids/refs/heads/v2/"
-      RFC = "https://raw.githubusercontent.com/relaton/relaton-data-rfcs/refs/heads/v2/"
-      RSS = "https://raw.githubusercontent.com/relaton/relaton-data-rfcsubseries/refs/heads/v2/"
+      # The combined corpus — RFCs, the RFC sub-series and Internet-Drafts in one
+      # repo, with one pubid `index-v2` covering all ~177k records (relaton#109).
+      # It replaces the three per-type repos this flavor used to read; those keep
+      # publishing their `index-v1` for released relatons, untouched.
+      IETF = "https://raw.githubusercontent.com/relaton/relaton-data-ietf/main/"
 
       # @param text [String]
-      # @return [RelatonIetf::IetfBibliographicItem]
+      # @return [Relaton::Ietf::ItemData, nil]
       def scrape_page(text)
-        # Remove initial "IETF " string if specified
-        ref = text.gsub(/^IETF /, "")
-        # ref.sub!(/(?<=^(?:RFC|BCP|FYI|STD))\s(\d+)/) { $1.rjust 4, "0" }
-        rfc_item ref
+        id = parse_id text.sub(/\AIETF\s+/, "")
+        return unless id
+
+        fetch_doc id
       rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError,
-            Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
-            Net::ProtocolError, SocketError
-        raise Relaton::RequestError, "No document found for #{ref} reference"
+             Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
+             Net::ProtocolError, SocketError
+        raise Relaton::RequestError, "No document found for #{text} reference"
       end
 
       private
 
-      # @param ref [String]
-      # @return [RelatonIetf::IetfBibliographicItem]
-      def rfc_item(ref) # rubocop:disable Metrics/MethodLength
-        case ref
-        when /^RFC/ then get_rfcs ref
-        when /^(?:BCP|FYI|STD)/ then get_rfcsubseries ref
-        when /^I-D/
-          ref.sub!(/^I-D[.\s]/, "")
-          get_ids ref
-        end
-      end
-
-      def get_rfcs(ref)
-        index = Relaton::Index.find_or_create :RFC, url: "#{RFC}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml"
-        row = index.search(ref).first
-        get_page "#{RFC}#{row[:file]}" if row
+      # The index stores parsed pubids, so the query has to be one too.
+      # `Type#search_candidates` narrows only when the query is not a String
+      # (`@file_io.sorted && id && !id.is_a?(String)`); a String falls through to
+      # `match_item`'s `item[:id].to_s.include?(id)`, which renders every pubid in
+      # the index on every lookup — measured at ~40 s per reference against the
+      # 177k-row index, versus sub-millisecond for a parsed one.
+      # `exact:` keeps the match exact. `Type#search` without it takes
+      # pubid's subset match, where a component the query omits is a wildcard —
+      # and an Internet-Draft slug omits the version, so `draft-foo` would match
+      # every `draft-foo-NN` and `.first` would return an arbitrary one (20,513
+      # such pairs in the first 40k rows of the index fixture).
+      def fetch_doc(id)
+        row = index.search(id, exact: true).first
+        get_page "#{IETF}#{row[:file]}" if row
       end
 
-      def get_rfcsubseries(ref)
-        index = Relaton::Index.find_or_create :RSS, url: "#{RSS}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml"
-        row = index.search(ref).first
-        get_page "#{RSS}#{row[:file]}" if row
+      def index
+        Relaton::Index.find_or_create(
+          :IETF, url: "#{IETF}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml",
+                 pubid_class: ::Pubid::Ietf::Identifier
+        )
       end
 
-      def get_ids(ref)
-        index = Relaton::Index.find_or_create :IDS, url: "#{IDS}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml"
-        row = index.search(ref).first
-        get_page "#{IDS}#{row[:file]}" if row
+      #
+      # Parse a reference into the identifier the index is keyed by.
+      #
+      # Normalises the two Internet-Draft spellings callers use — `I-D.<slug>`
+      # and `I-D <slug>` — onto the `draft-…` form pubid parses and the index
+      # stores. The bare `I-D.ietf-quic-transport` spelling (the bibxml anchor,
+      # and the `docnumber` IETF records carry) gains the `draft-` stem: the old
+      # plain-string index matched it by substring, and matching is exact now.
+      #
+      # The capture is `(\S.*)`, not `(.+)`: it must start non-space so it cannot
+      # overlap the preceding `\s*`. Behaviour is unchanged on every real
+      # reference — with `(.+)` the greedy `\s*` only ever yields ground on an
+      # all-whitespace remainder, which pubid rejects anyway — but the
+      # unambiguous form is about twice as fast on a pathological input and is
+      # what CodeQL's rb/polynomial-redos models. Measured before changing it:
+      # the old form was already linear (0.66 ms at 80k chars), so this is
+      # clarity, not a ReDoS fix.
+      #
+      # @param ref [String]
+      # @return [Pubid::Ietf::Identifier, nil] nil when pubid has no grammar for
+      #   it, so an out-of-flavor reference logs "Not found." instead of raising
+      #
+      def parse_id(ref)
+        if (draft = ref[/\AI-D[.\s]\s*(\S.*)\z/m, 1])
+          ref = draft.start_with?("draft-") ? draft : "draft-#{draft}"
+        end
+        ::Pubid::Ietf::Identifier.parse ref
+      rescue StandardError => e
+        # Logged, not swallowed: this repo git-pins pubid to a moving `main`,
+        # so a grammar regression would otherwise present as every IETF
+        # reference quietly reporting "Not found."
+        Util.debug "`#{ref}` is not an IETF identifier: #{e.message}"
+        nil
       end
 
       # @param uri [String]
-      # @return [RelatonIetf::IetfBibliographicItem, nil] HTTP response body
+      # @return [Relaton::Ietf::ItemData, nil] HTTP response body
       def get_page(uri)
         res = Net::HTTP.get_response(URI(uri))
         return unless res.code == "200"

data/lib/relaton/ietf.rb

@@ -14,7 +14,10 @@ require_relative "ietf/bibliography"
 
 module Relaton
   module Ietf
-    INDEXFILE = "index-v1".freeze
+    # The pubid-structured index, written by DataFetcher and read by Scraper
+    # (relaton#109). One constant again now that both sides are on it — the
+    # second one existed only while producer and consumer were split.
+    INDEXFILE = "index-v2".freeze
     # Returns hash of XML reammar
     # @return [String]
     def self.grammar_hash

data/lib/relaton/iho/bibliography.rb

@@ -89,7 +89,7 @@ module Relaton
             :iho,
             url: "#{ENDPOINT}#{INDEXFILE}.zip",
             file: "#{INDEXFILE}.yaml",
-            pubid_class: ::Pubid::Iho::Identifiers::Base,
+            pubid_class: ::Pubid::Iho::Identifier,
           )
         end
 

data/lib/relaton/iho/docidentifier.rb

@@ -24,7 +24,7 @@ module Relaton
       end
 
       def to_all_parts!
-        @pubid&.all_parts = true
+        @pubid &&= @pubid.to_all_parts
       end
     end
   end