relaton 3.0.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/itu/data_merge_r.rb

@@ -0,0 +1,149 @@
+require_relative "../itu"
+
+module Relaton
+  module Itu
+    # Incremental write path for harvested ITU-R records (issue #75).
+    #
+    # The ITU-R dataset cannot be rebuilt from scratch — DataCrawlerR reads the
+    # pages ITU still serves, and those expose **less** than the decommissioned
+    # RunSearch feed did. Two consequences drive this module:
+    #
+    # 1. **A published date must never be rewritten.** A Recommendation page
+    #    carries only its approval date, while the preserved record holds the
+    #    publication date RunSearch served — measured on the whole BO series,
+    #    those differ for every record and the *year* differs for 59% of them.
+    #    Overwriting would change which edition answers a dated reference. (A
+    #    Report page does carry the publication date, so reports reproduce it;
+    #    the rule costs nothing there.)
+    # 2. **A harvest is a partial view.** Only the families the crawler
+    #    implements are seen, so a run must add and backfill, never delete, and
+    #    must leave untouched records byte-identical so the data repo's diff
+    #    shows only what actually changed.
+    #
+    # Merge rules, applied per record: `date` is kept as published, always;
+    # `title` and `source` are filled in only when the published record has
+    # none; everything else is left alone. A record whose published counterpart
+    # has a **different doctype** is a collision — an ITU-R report and
+    # recommendation in the same series can share a docidentifier, and therefore
+    # a filename — so it is reported and skipped rather than silently
+    # overwriting the other document.
+    module DataMergeR
+      extend self
+
+      #
+      # Merge harvested records into the dataset the fetcher writes to.
+      #
+      # @param items [Array<Relaton::Itu::ItemData>] as DataCrawlerR#harvest returns
+      # @param fetcher [Relaton::Itu::DataFetcher] supplies output_file/write_file
+      #   and the index, so harvested records get the same pubid guard and
+      #   unparseable-id reporting as the ITU-T harvest
+      #
+      # @return [Hash] { added:, backfilled:, unchanged:, skipped:,
+      #   collisions: [[file, published_doctype, harvested_doctype], …] }
+      #
+      def write_all(items, fetcher)
+        stats = { added: 0, backfilled: 0, unchanged: 0, skipped: 0, collisions: [] }
+        seen = {}
+        items.each do |bib|
+          id = primary_id(bib)
+          unless id
+            stats[:skipped] += 1
+            Util.error "ITU-R merge: record with no primary docidentifier skipped"
+            next
+          end
+
+          file = fetcher.output_file id
+          # First writer of a filename keeps it; a later claimant is reported and
+          # dropped, never merged into a document it isn't.
+          if (other = seen[file])
+            record_collision stats, file, other, "#{id} (#{doctype bib})", "this harvest"
+            next
+          end
+          seen[file] = "#{id} (#{doctype bib})"
+          merge_one bib, id, file, fetcher, stats
+        rescue => e # rubocop:disable Style/RescueStandardError
+          # One unreadable published file must not discard an hour of crawling.
+          stats[:skipped] += 1
+          Util.error "ITU-R merge: #{id || '(no id)'} skipped: #{e.message}"
+        end
+        stats
+      end
+
+      private
+
+      # @return [void]
+      def merge_one(bib, id, file, fetcher, stats)
+        unless File.exist? file
+          fetcher.write_file bib
+          stats[:added] += 1
+          return
+        end
+
+        published = Item.from_yaml File.read(file, encoding: "UTF-8")
+        published_id = primary_id published
+        # Both halves matter: a different doctype means a report and a
+        # recommendation share a filename, and a different id means two docids
+        # sanitize to one filename (`output_file` collapses `.`, `/` and spaces
+        # alike, so `ITU-R BO.4/BL/4` and `ITU-R BO.4-BL-4` would meet here).
+        if doctype(published) != doctype(bib) || published_id != id
+          record_collision stats, file, "#{published_id} (#{doctype published})",
+                           "#{id} (#{doctype bib})", "the published dataset"
+          return
+        end
+
+        if backfill published, bib
+          fetcher.write_file published
+          stats[:backfilled] += 1
+        else
+          # Not rewritten: an identical re-serialization would churn the data
+          # repo's diff. Index it anyway — the index is rebuilt from scratch on
+          # every run, so a record that is not written still has to be in it.
+          # Indexed by the id the *file* carries, as #index_files does.
+          fetcher.index_primary published_id, file
+          stats[:unchanged] += 1
+        end
+      end
+
+      # Fill only what the published record is missing. `date` is deliberately
+      # absent from this list — see the module comment.
+      #
+      # @param published [Relaton::Itu::Item] mutated in place
+      # @param bib [Relaton::Itu::ItemData] the harvested record
+      # @return [Boolean] whether anything was filled in
+      def backfill(published, bib)
+        filled = false
+        if published.source.nil? || published.source.empty?
+          published.source = bib.source
+          filled ||= !(bib.source.nil? || bib.source.empty?)
+        end
+        if published.title.nil? || published.title.empty?
+          published.title = bib.title
+          filled ||= !(bib.title.nil? || bib.title.empty?)
+        end
+        filled
+      end
+
+      # @return [void]
+      def record_collision(stats, file, kept, dropped, where)
+        stats[:skipped] += 1
+        stats[:collisions] << [file, kept, dropped]
+        Util.error "ITU-R merge: filename collision on #{file} — #{kept} in #{where} vs " \
+                   "harvested #{dropped}; not overwritten"
+      end
+
+      # Strictly the primary docid: DataFetcher#write_file reads
+      # `docidentifier.find(&:primary).content` with no fallback, so a record
+      # without one has nowhere to be written and is skipped here instead.
+      #
+      # @return [String, nil]
+      def primary_id(bib)
+        bib.docidentifier.find(&:primary)&.content
+      end
+
+      # @return [String, nil]
+      def doctype(bib)
+        bib.ext&.doctype&.content
+      end
+    end
+  end
+end

data/lib/relaton/itu/data_parser_r.rb

@@ -1,140 +1,214 @@
 module Relaton
   module Itu
+    # Map one normalized ITU-R row to an ItemData.
+    #
+    # The row comes from DataCrawlerR's three-level `/pub` + `/rec` crawl (issue
+    # #75), which replaced the decommissioned RunSearch result hash this module
+    # used to consume. The split mirrors DataParserT: the crawler owns the HTTP
+    # and the page scraping, this module owns the mapping, so it stays a pure
+    # unit with no network in its specs. Row keys:
+    #
+    #   :id      "R-REC-BO.1130-5-202602-I"  page id
+    #   :code    "BO.1130-5 (02/2026)"       displayed code -> primary docid
+    #   :title   English title
+    #   :status  "In force (Main)" | "Superseded" | …   (scraped, not modelled)
+    #   :date    "2026-02-18" | "2026-02"    approval date for a Recommendation,
+    #                                        publication date for a Report
+    #   :url     the record's landing page
+    #   :pdf     absolute dms_pubrec/dms_pub URL
+    #   :family  "R-REC" | "R-REP" | "R-QUE" | "R-RES" | "R-HDB"
     module DataParserR
       extend self
 
-      TYPE_MAP = {
-        "ITU-R Recommendations" => "recommendation",
-        "ITU-R Questions" => "question",
-        "ITU-R Reports" => "technical-report",
-        "Handbooks" => "handbook",
-        "ITU-R Resolutions" => "resolution",
+      FAMILY_DOCTYPE = {
+        "R-REC" => "recommendation",
+        "R-REP" => "technical-report",
+        "R-QUE" => "question",
+        "R-RES" => "resolution",
+        "R-HDB" => "handbook",
       }.freeze
 
       #
-      # Parse ITU-R document from search API result.
+      # Parse an ITU-R document from a normalized crawler row.
       #
-      # @param result [Hash] single search result from the API
+      # @param row [Hash] see the key list above
+      # @param errors [Hash] shared error tally, `&&=`-narrowed per field so a
+      #   field that succeeded once is never reported as missing
       #
-      # @return [Relaton::Itu::ItemData] bibliographic item
+      # @return [Relaton::Itu::ItemData, nil] nil when the family is unknown
       #
-      def parse(result, errors = {})
-        @errors = errors
-        doctype = fetch_doctype(result)
+      def parse(row, errors = {})
+        doctype = fetch_doctype(row, errors)
         return unless doctype
 
+        docid = fetch_docid(row, errors)
+        # A record with no primary docid can't be written or indexed —
+        # DataFetcher#write_file reads `docidentifier.find(&:primary).content` —
+        # so drop it here, as DataParserT does.
+        return if docid.empty?
+
         Relaton::Itu::ItemData.new(
-          docidentifier: fetch_docid(result), title: fetch_title(result),
-          date: fetch_date(result), language: ["en"],
-          source: fetch_source(result), script: ["Latn"],
+          docidentifier: docid, title: fetch_title(row, errors),
+          date: fetch_date(row, errors), language: ["en"],
+          source: fetch_source(row, errors), script: ["Latn"],
           type: "standard", ext: Relaton::Itu::Ext.new(doctype: doctype, flavor: "itu"),
         )
       end
 
-      # @param result [Hash]
+      # Each ITU-R family spells its identifier differently, and the published
+      # dataset is the authority on which spelling. Verified against
+      # relaton-data-itu:
+      #
+      #   R-REC  the displayed code    "BO.1130-5 (02/2026)" -> ITU-R BO.1130-5
+      #   R-REP  the same, prefixed    "BO.1227-2 (1998)"    -> Report ITU-R BO.1227-2
+      #   R-QUE  the same, + a colon   "202-2/1"             -> ITU-R 202-2/1:
+      #   R-RES  the **page id**       R-RES-R.1-9-2023      -> ITU-R R.1-9
+      #   R-HDB  the **page id**       R-HDB-43-2013         -> ITU-R 43.HDB (2013)
+      #
+      # Three of those need saying out loud. A **Report** is prefixed because
+      # Recommendations and Reports number independently, so `ITU-R BT.2020-1`
+      # alone names two different documents (pubid #327 types the prefixed form
+      # `pubid:itu:report`, and `output_file` sends it to `report-itu-r-*.yaml`
+      # instead of colliding). A **Resolution** cannot use its displayed code at
+      # all: the page renders "Res.1-9 (2023)" while the published record is
+      # `ITU-R R.1-9`, which only the id carries. A **Handbook** likewise: the
+      # index lists its *title* and the leaf shows only a year — see
+      # #handbook_docid.
+      #
+      # The code is NBSP-folded first — the cells are padded with ` `, which
+      # neither `\s` nor String#strip match, and an NBSP left in a docid
+      # sanitizes into the filename and defeats `Pubid::Itu`.
+      #
+      # @param row [Hash]
+      # @return [String, nil] nil when the row carries nothing to build from
+      def family_docid(row)
+        # " " spelled as an escape on purpose: a literal NBSP here is invisible
+        # and has been silently lost by tooling before.
+        #
+        # The date suffix is cut at the bare `(`, leaving the space before it to
+        # #strip. A leading `\s*` would make this polynomial-time (CodeQL
+        # rb/polynomial-redos): unanchored, it can start matching anywhere in a
+        # run of spaces, so a code that never reaches a `(` costs one pass per
+        # space. Same result, no ambiguity.
+        code = row[:code].to_s.tr(" ", " ").sub(/\(.*\z/m, "").strip
+        # The two id-derived families, before `code` gets a say.
+        return "ITU-R #{resolution_number row}" if row[:family] == "R-RES"
+        return handbook_docid(row) if row[:family] == "R-HDB"
+        return nil if code.empty?
+
+        case row[:family]
+        when "R-REP" then "Report ITU-R #{code}"
+        when "R-QUE" then "ITU-R #{code}:"
+        else "ITU-R #{code}"
+        end
+      end
+
+      # "R-HDB-43-2013" -> "ITU-R 43.HDB (2013)".
+      #
+      # The displayed code is unusable twice over: the index cell holds the
+      # handbook's *title*, and the leaf holds a year that can disagree with its
+      # own id (`R-HDB-43-2013` displays "2014"). The id is the only consistent
+      # source, so the year comes from there.
+      #
+      # Keyed **per edition** because ITU publishes several per handbook (43 has
+      # 2002, 2013 and 2026) and `Pubid::Itu` models the year, so they stay
+      # distinct records. The bare `ITU-R 43` that four published records carry
+      # parses as a *recommendation*, claiming a number that is not its own —
+      # this is what replaces it.
+      #
+      # @param row [Hash]
+      # @return [String, nil]
+      def handbook_docid(row)
+        m = row[:id].to_s.match(/\AR-HDB-(?<number>\d+)(?:-(?<year>(?:19|20)\d{2}))?/)
+        return nil unless m
+
+        m[:year] ? "ITU-R #{m[:number]}.HDB (#{m[:year]})" : "ITU-R #{m[:number]}.HDB"
+      end
+
+      # "R-RES-R.1-9-2023" -> "R.1-9"
+      #
+      # @param row [Hash]
+      # @return [String, nil]
+      def resolution_number(row)
+        n = row[:id].to_s.sub(/\AR-RES-/, "").sub(/-(?:19|20)\d{2}\z/, "")
+        n.empty? ? nil : n
+      end
+
+      # @param row [Hash]
       # @return [Array<Relaton::Bib::Docidentifier>]
-      def fetch_docid(result)
-        title = result["Title"].to_s
-        id = title.match(/^(ITU-R\s+\S+)/)&.captures&.first
-        return(@errors[:docid] &&= true; []) unless id
-
-        id = id.sub(/\s*\(.*/, "")
-        result_ids = [Docidentifier.new(type: "ITU", content: id, primary: true)]
-        @errors[:docid] &&= result_ids.empty?
-        result_ids
+      def fetch_docid(row, errors = {})
+        content = family_docid row
+        if content.nil? || content == "ITU-R "
+          errors[:docid] &&= true
+          return []
+        end
+
+        r = [Docidentifier.new(type: "ITU", content: content, primary: true)]
+        errors[:docid] &&= r.empty?
+        r
       end
 
-      # @param result [Hash]
+      # @param row [Hash]
       # @return [Array<Relaton::Bib::Title>]
-      def fetch_title(result)
-        title = result["Title"].to_s
-        content = title.sub(/^[^:]+:\s*/, "").strip
-        content = title unless content.length > 0
+      def fetch_title(row, errors = {})
+        content = row[:title].to_s.strip
+        if content.empty?
+          errors[:title] &&= true
+          return []
+        end
+
         r = [Relaton::Bib::Title.new(type: "main", content: content, language: "en", script: "Latn")]
-        @errors[:title] &&= r.empty?
+        errors[:title] &&= r.empty?
         r
       end
 
-      # @param result [Hash]
+      # Whatever date the crawler could see: for a **Recommendation** that is the
+      # approval date — *not* the publication date the preserved records carry,
+      # which died with RunSearch (issue #75), which is why DataMergeR never
+      # rewrites one — and for a **Report** it is the publication date itself,
+      # read off the edition page's posted files. Day precision when it comes
+      # from the page, month or year precision when derived from the page id.
+      #
+      # @param row [Hash]
       # @return [Array<Relaton::Bib::Date>]
-      def fetch_date(result)
-        prop = property(result, "Publication date")
-        unless prop
-          @errors[:date] &&= true
-          return []
-        end
-
-        date = parse_pub_date(prop)
-        unless date
-          @errors[:date] &&= true
+      def fetch_date(row, errors = {})
+        date = row[:date].to_s.strip
+        if date.empty?
+          errors[:date] &&= true
           return []
         end
 
         r = [Relaton::Bib::Date.new(type: "published", at: date)]
-        @errors[:date] &&= r.empty?
+        errors[:date] &&= r.empty?
         r
       end
 
-      # @param result [Hash]
+      # @param row [Hash]
       # @return [Array<Relaton::Bib::Uri>]
-      def fetch_source(result)
-        locations = result["Locations"]
-        unless locations.is_a?(Array)
-          @errors[:source] &&= true
+      def fetch_source(row, errors = {})
+        pdf = row[:pdf].to_s.strip
+        if pdf.empty?
+          errors[:source] &&= true
           return []
         end
 
-        pdf = locations.find { |l| l["Type"] == "pdf" }
-        unless pdf && pdf["RawHref"]
-          @errors[:source] &&= true
-          return []
-        end
-
-        r = [Relaton::Bib::Uri.new(type: "pdf", content: pdf["RawHref"])]
-        @errors[:source] &&= r.empty?
+        r = [Relaton::Bib::Uri.new(type: "pdf", content: pdf)]
+        errors[:source] &&= r.empty?
         r
       end
 
-      # @param result [Hash]
+      # @param row [Hash]
       # @return [Relaton::Itu::Doctype, nil]
-      def fetch_doctype(result)
-        type_value = property(result, "Type")
-        mapped = TYPE_MAP[type_value]
+      def fetch_doctype(row, errors = {})
+        mapped = FAMILY_DOCTYPE[row[:family]]
         unless mapped
-          @errors[:doctype] &&= true
+          errors[:doctype] &&= true
           return
         end
 
-        @errors[:doctype] &&= false
+        errors[:doctype] &&= false
         Doctype.new(content: mapped)
       end
-
-      private
-
-      # Find a property value from the result's Properties array.
-      # @param result [Hash]
-      # @param name [String]
-      # @return [String, nil]
-      def property(result, name)
-        props = result["Properties"]
-        return unless props.is_a?(Array)
-
-        entry = props.find { |p| p["Title"] == name }
-        entry&.[]("Value")
-      end
-
-      # Parse publication date string like "January, 2024" or "2024".
-      # @param value [String]
-      # @return [String, nil]
-      def parse_pub_date(value)
-        case value
-        when /(\w+),?\s+(\d{4})/
-          month = Date::MONTHNAMES.index($1)
-          month ? "#{$2}-#{format('%02d', month)}" : $2
-        when /(\d{4})/
-          $1
-        end
-      end
     end
   end
 end

data/lib/relaton/itu/data_parser_t.rb

@@ -0,0 +1,228 @@
+require_relative "../itu"
+require_relative "recommendation_parser"
+
+module Relaton
+  module Itu
+    # Parse a single row of the ITU-T recommendation index returned by the
+    # mws/api/recommendations/searchRecs endpoint (see DataFetcher#search_recs)
+    # into an ItemData. Recommendations and supplements share one row schema and
+    # are distinguished by "Suppl" in rec_name; each edition is its own row, so
+    # the docid keeps the (MM/YYYY) edition date (e.g. "ITU-T A.1 (10/2000)") —
+    # which Pubid::Itu parses per edition, keeping index rows and filenames
+    # unique across editions. Mirrors DataParserR (the ITU-R row parser).
+    #
+    # The searchRecs row is metadata-thin (docid/title/date/source/doctype). When
+    # an `agent` is given, each record is **enriched** with the same
+    # `getRecHdrDetail`-sourced fields the live runtime path emits — abstract,
+    # ISO/IEC co-identifier, editorial-group contributors, status — via the shared
+    # `RecommendationFields` (through `RecommendationParser`), so the harvested
+    # record matches a live `Bibliography.get`. Enrichment is best-effort: a
+    # detail-fetch failure degrades to the thin record rather than losing it.
+    module DataParserT
+      extend self
+
+      # rec_name markers → Doctype. Ordered; first match wins. A plain edition
+      # (no marker) is a "recommendation". Values are members of Doctype::TYPES.
+      DOCTYPE_MARKERS = {
+        /\bSuppl\b/ => "recommendation-supplement",
+        /\bAmd\b/ => "recommendation-amendment",
+        /\bCor\b/ => "recommendation-corrigendum",
+        /\bAnnex\b/ => "recommendation-annex",
+      }.freeze
+
+      # The `errors` hash is threaded through the fetch_* helpers rather than
+      # held in an ivar: this module is `extend self`, so an ivar would be state
+      # shared by every caller — and DataFetcher#fetch_recommendations parses
+      # rows from a pool of threads.
+      #
+      # @param row [Hash] single row from searchRecs Data
+      # @param agent [Mechanize, nil] when present, enrich via getRecHdrDetail
+      # @param errors [Hash]
+      # @return [Relaton::Itu::ItemData, nil]
+      def parse(row, agent = nil, errors = {}, cache: NullCache.instance)
+        docid = fetch_docid(row, errors)
+        return if docid.empty?
+
+        enr = enrichment(row, agent, cache)
+        date = fetch_date(row, errors)
+        row_source = fetch_source(row, errors)
+        Relaton::Itu::ItemData.new(
+          docidentifier: docid + enr.fetch(:iso, []),
+          title: fetch_title(row, errors),
+          edition: enr[:edition],
+          abstract: enr.fetch(:abstract, []),
+          date: date, language: ["en"],
+          status: enr[:status],
+          relation: enr.fetch(:relation, []),
+          contributor: enr.fetch(:contributor, []),
+          copyright: fetch_copyright(date),
+          place: [Relaton::Bib::Place.new(city: "Geneva")],
+          source: enr[:source] || row_source, script: ["Latn"],
+          type: "standard",
+          ext: Relaton::Itu::Ext.new(doctype: fetch_doctype(row, errors), flavor: "itu"),
+        )
+      end
+
+      # Best-effort getRecHdrDetail enrichment for one record. Returns {} when
+      # there is no agent or the detail fetch fails, so the record still gets its
+      # thin fields. ITU-T records are always sector T → abbreviation "ITU",
+      # bureau "T".
+      #
+      # @param row [Hash]
+      # @param agent [Mechanize, nil]
+      # @param cache [#fetch, #warm] shared across the worker pool, so the
+      #   family-invariant endpoints are fetched once per recommendation rather
+      #   than once per edition. Like `errors`, it is threaded through as a
+      #   parameter rather than held as state — DataParserT is `extend self`.
+      #   The source is the handle URI and the PDF URI, as on the live path. It
+      #   is nil when the header has no handle_id, so the row source is used.
+      # @return [Hash] { iso:, abstract:, status:, edition:, relation:,
+      #   contributor:, source: }
+      def enrichment(row, agent, cache = NullCache.instance)
+        return {} unless agent && row["idrec"]
+
+        f = RecommendationParser.new(agent, row["idrec"], false, cache: cache)
+        ed = f.fetch_edition
+        {
+          iso: Array(f.iso_docid),
+          abstract: f.fetch_abstract,
+          status: f.fetch_status,
+          edition: (Relaton::Bib::Edition.new(content: ed) if ed),
+          relation: f.fetch_relations,
+          contributor: [f.publisher("ITU"), f.editorial_group("T")].compact,
+          source: (f.fetch_source if f.doc["handle_id"]),
+        }
+      rescue StandardError => e
+        Util.warn "ITU-T enrichment failed for idrec=#{row['idrec']}: #{e.message}"
+        {}
+      end
+
+      # The ITU copyright, from the record's own publication year — no detail
+      # fetch needed, so a thin (un-enriched) record carries it too. Mirrors
+      # `Scraper#fetch_copyright` on the live path.
+      #
+      # @param date [Array<Relaton::Bib::Date>] the record's dates
+      # @return [Array<Relaton::Bib::Copyright>]
+      def fetch_copyright(date)
+        year = date.first&.at.to_s[/\d{4}/]
+        return [] unless year
+
+        org = Relaton::Bib::Organization.new(
+          name: [Relaton::Bib::TypedLocalizedString.new(content: "International Telecommunication Union")],
+          abbreviation: Relaton::Bib::LocalizedString.new(content: "ITU"),
+          uri: [Relaton::Bib::Uri.new(content: "www.itu.int")],
+        )
+        owner = [Relaton::Bib::ContributionInfo.new(organization: org)]
+        [Relaton::Bib::Copyright.new(from: year, owner: owner)]
+      end
+
+      # The primary docid is the rec_name prefixed with "ITU-T", keeping the
+      # trailing " (MM/YYYY)" edition date so Pubid::Itu identifies each edition
+      # distinctly (matches the existing "ITU-T L.163 (11/2018)" convention).
+      #
+      # @param row [Hash]
+      # @return [Array<Relaton::Itu::Docidentifier>]
+      def fetch_docid(row, errors = {})
+        name = normalize_rec_name row["rec_name"]
+        if name.empty?
+          errors[:docid] &&= true
+          return []
+        end
+
+        r = [Docidentifier.new(type: "ITU", content: "ITU-T #{name}", primary: true)]
+        errors[:docid] &&= r.empty?
+        r
+      end
+
+      # One rule, for the one spelling `::Pubid::Itu` deliberately will not take:
+      # a space where the series dot belongs ("G 231 (10/1976)"). pubid rejects it
+      # on purpose, because a space there is ambiguous against the series-only
+      # supplement form ("G Suppl. 1"), so relaton canonicalises it rather than
+      # leaving 4 records unindexed. `#index_primary` drops ids pubid rejects, and
+      # a dropped record is written but never indexed — reachable only through the
+      # live `rec.aspx` fallback.
+      #
+      # This changes **no filename**: `Core::DataFetcher#output_file` collapses
+      # runs of whitespace and punctuation alike, so `"G 231 (10/1976)"` and
+      # `"G.231 (10/1976)"` both yield `itu-t-g-231-10-1976.yaml`. The affected
+      # records simply gain an index row.
+      #
+      # Everything else searchRecs spells oddly — the doubled space, `Suppl.1` run
+      # into its number, and bare `v10`/`V2`/`v.1` versions — **parses natively**
+      # as of pubid #325, so the rules that used to rewrite those are gone: pubid
+      # now canonicalises on `to_s` itself and ITU's own spelling is kept in the
+      # docid. Forms pubid cannot parse are still left alone; guessing a spelling
+      # would invent identifiers rather than canonicalise them.
+      #
+      # @param rec_name [String, nil]
+      # @return [String]
+      def normalize_rec_name(rec_name)
+        rec_name.to_s.strip.sub(/\A([A-Z]+) (\d)/, '\1.\2')
+      end
+
+      # @param row [Hash]
+      # @return [Array<Relaton::Bib::Title>]
+      def fetch_title(row, errors = {})
+        content = row["title"].to_s.strip
+        r = content.empty? ? [] : [Relaton::Bib::Title.new(type: "main", content: content, language: "en", script: "Latn")]
+        errors[:title] &&= r.empty?
+        r
+      end
+
+      # Prefer a full day-precision approval_date (YYYY-MM-DD, matching the live
+      # runtime path); fall back to the (MM/YYYY) edition date in rec_name; and
+      # only then to a coarser (year / year-month) approval_date. Ordering this way
+      # means a coarse or malformed approval_date never downgrades precision below
+      # the always-well-formed rec_name edition date.
+      #
+      # @param row [Hash]
+      # @return [Array<Relaton::Bib::Date>]
+      def fetch_date(row, errors = {})
+        approval = row["approval_date"].to_s
+        date = approval[/\d{4}-\d{2}-\d{2}/] || rec_name_date(row["rec_name"]) || approval[/\d{4}(-\d{2})?/]
+        if date.nil? || date.empty?
+          errors[:date] &&= true
+          return []
+        end
+
+        r = [Relaton::Bib::Date.new(type: "published", at: date)]
+        errors[:date] &&= r.empty?
+        r
+      end
+
+      # @param row [Hash]
+      # @return [Array<Relaton::Bib::Uri>]
+      def fetch_source(row, errors = {})
+        link = row["dms_link"].to_s.strip
+        if link.empty? || link == "-"
+          errors[:source] &&= true
+          return []
+        end
+
+        r = [Relaton::Bib::Uri.new(type: "src", content: link)]
+        errors[:source] &&= r.empty?
+        r
+      end
+
+      # @param row [Hash]
+      # @return [Relaton::Itu::Doctype]
+      def fetch_doctype(row, errors = {})
+        name = row["rec_name"].to_s
+        content = DOCTYPE_MARKERS.find { |re, _| name.match?(re) }&.last || "recommendation"
+        errors[:doctype] &&= false
+        Doctype.new(content: content)
+      end
+
+      private
+
+      # "A.1 (10/2000)" -> "2000-10"
+      # @param name [String, nil]
+      # @return [String, nil]
+      def rec_name_date(name)
+        return unless name =~ %r{\((\d{2})/(\d{4})\)}
+
+        "#{$2}-#{$1}"
+      end
+    end
+  end
+end