relaton 3.0.0.pre.alpha.4 → 3.0.0.pre.alpha.5

data/lib/relaton/iso/data_parser.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "nokogiri"
+require "moxml"
 require_relative "../iso"
 require_relative "scraper"
 
@@ -331,7 +331,7 @@ module Relaton
       end
 
       def strip_html(html)
-        Nokogiri::HTML.fragment(html).text.strip.gsub(/\s+/, " ")
+        Moxml.new.parse_fragment(html).map(&:text).join.strip.gsub(/\s+/, " ")
       end
 
       # ---- copyright -------------------------------------------------------

data/lib/relaton/iso/processor.rb

@@ -39,6 +39,11 @@ module Relaton
         DataFetcher.fetch(source, **opts)
       end
 
+      # `Bibliography.get` applies a year to the document the reference names.
+      def fold_year(pubid, year)
+        fold_year_on_root pubid, year
+      end
+
       # @param xml [String]
       # @return [RelatonIsoBib::IsoBibliographicItem]
       def from_xml(xml)

data/lib/relaton/iso/scraper.rb

@@ -95,7 +95,7 @@ module Relaton
       def id
         return @id if defined?(@id)
 
-        did = @doc.at("//h1/span[1]")
+        did = @doc.at_xpath("//h1/span[1]")
         @errors[:id] &&= did.nil?
         @id = did && did.text.split(" | ").first.strip
       end
@@ -117,7 +117,7 @@ module Relaton
       def edition
         return @edition if defined?(@edition)
 
-        ed = @doc.at("//div[div[.='Edition']]/text()[last()]")
+        ed = @doc.at_xpath("//div[div[.='Edition']]/text()[last()]")
         @errors[:edition] &&= ed.nil?
         @edition = ed && Bib::Edition.new(content: ed.text.match(/\d+$/).to_s)
       end
@@ -172,7 +172,7 @@ module Relaton
         langs = languages.each_with_object([]) do |l, s|
           # Don't need to get page for en. We already have it.
           d = l[:path] ? get_page(l[:path])[0] : @doc
-          unless d.at("//h5[@class='help-block'][.='недоступно на русском языке']")
+          unless d.at_xpath("//h5[@class='help-block'][.='недоступно на русском языке']")
             s << l
             titles += fetch_title(d, l[:lang])
 
@@ -208,7 +208,7 @@ module Relaton
 
       # Get page.
       # @param path [String] page's path
-      # @return [Array<Nokogiri::HTML::Document, String>]
+      # @return [Array<Moxml::Document, String>]
       def get_page(path) # rubocop:disable Metrics/MethodLength
         try = 0
         begin
@@ -276,12 +276,12 @@ module Relaton
       # @param [Net::HTTPOK] resp HTTP response
       # @param [URI::HTTPS] uri URI of the page
       #
-      # @return [Nokogiri::HTML4::Document] document
+      # @return [Moxml::Document] document
       # @raise [Relaton::RequestError] if the page could not be parsed
       #
       def try_if_fail(resp, uri)
         10.times do
-          doc = Nokogiri::HTML(resp.body)
+          doc = Moxml.new.parse_html(resp.body)
           # stop trying if page has a document id
           return doc if item_ref(doc)
 
@@ -312,12 +312,12 @@ module Relaton
       #
       # Parse ID from the document.
       #
-      # @param [Nokogiri::HTML::Document] doc document to parse
+      # @param [Moxml::Document] doc document to parse
       #
       # @return [String, nil] ID
       #
       def item_ref(doc)
-        ref = doc.at("//main//section/div/div/div//h1/span[1]")
+        ref = doc.at_xpath("//main//section/div/div/div//h1/span[1]")
         @errors[:reference] &&= ref.nil?
         ref&.text&.strip
       end
@@ -334,7 +334,7 @@ module Relaton
       def stage_code
         return @stage_code if defined?(@stage_code)
 
-        stc = @doc.at("//ul[@class='dropdown-menu']/li[@class='active']/a/span[@class='stage-code']")
+        stc = @doc.at_xpath("//ul[@class='dropdown-menu']/li[@class='active']/a/span[@class='stage-code']")
         @errors[:stage] &&= stc.nil?
         @stage_code = stc&.text
       end
@@ -351,7 +351,7 @@ module Relaton
         rels = @doc.xpath(
           "//ul[@class='steps']/li", "//div[contains(@class, 'sub-step')]"
         ).reduce([]) do |a, r|
-          type, date = relation_type(r.at("h4", "h5").text.strip)
+          type, date = relation_type(r.at_xpath("h4|h5").text.strip)
           next a if types.include?(type)
 
           a + create_relations(r, type, date)
@@ -383,7 +383,7 @@ module Relaton
       #
       # Create relations.
       #
-      # @param [Nokogiri::HTML::Element] rel relation element
+      # @param [Moxml::Element] rel relation element
       # @param [String] type relation type
       # @param [Hash{Symbol=>String}] date relation document date
       # @option date [String] :type date type
@@ -400,7 +400,7 @@ module Relaton
       end
 
       # Fetch titles.
-      # @param doc [Nokogiri::HTML::Document]
+      # @param doc [Moxml::Document]
       # @param lang [String]
       # @return [Array<RelatonBib::TypedTitleString>]
       def fetch_title(doc, lang) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
@@ -417,7 +417,7 @@ module Relaton
       end
 
       def parse_titles(doc)
-        # head = doc.at "//nav[contains(@class,'heading-condensed')]"
+        # head = doc.at_xpath "//nav[contains(@class,'heading-condensed')]"
         ttls = doc.xpath("//h1[@class='stdTitle']/span[position()>1]").map(&:text)
         return ttls if @errors[:title] &&= ttls.empty?
 
@@ -455,14 +455,14 @@ module Relaton
       def fetch_dates # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
         dates = []
         %r{^[^\s]+\s[\d-]+:(?<ref_date_str>\d{4})} =~ id
-        pub_date_str = @doc.at("//span[@itemprop='releaseDate']")
+        pub_date_str = @doc.at_xpath("//span[@itemprop='releaseDate']")
         @errors[:date_pub] &&= pub_date_str.nil?
         if ref_date_str
           dates += parse_date_from_id ref_date_str, pub_date_str
         elsif pub_date_str
           dates << Bib::Date.new(type: "published", at: pub_date_str.text)
         end
-        corr_data = @doc.at "//span[@itemprop='dateModified']"
+        corr_data = @doc.at_xpath "//span[@itemprop='dateModified']"
         @errors[:date_corr] &&= corr_data.nil?
         dates << Bib::Date.new(type: "corrected", at: corr_data.text) if corr_data
         dates
@@ -522,13 +522,13 @@ module Relaton
       #
       def fetch_source(url) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength
         source = [Bib::Uri.new(type: "src", content: url)]
-        obp = @doc.at("//a[.='Read sample']")
+        obp = @doc.at_xpath("//a[.='Read sample']")
         @errors[:link_obp] &&= obp.nil?
         source << Bib::Uri.new(type: "obp", content: obp[:href]) if obp
-        rss = @doc.at("//a[contains(@href, 'rss')]")
+        rss = @doc.at_xpath("//a[contains(@href, 'rss')]")
         @errors[:link_rss] &&= rss.nil?
         source << Bib::Uri.new(type: "rss", content: DOMAIN + rss[:href]) if rss
-        pub = @doc.at "//p[contains(., 'publicly available')]/a",
+        pub = @doc.at_xpath "//p[contains(., 'publicly available')]/a",
                       "//p[contains(., 'can be downloaded from the')]/a"
         @errors[:link_pub] &&= pub.nil?
         source << Bib::Uri.new(type: "pub", content: pub[:href]) if pub
@@ -542,7 +542,7 @@ module Relaton
         owner_name = ref.match(/.*?(?=\s)/).to_s
         from = ref.match(/(?<=:)\d{4}/).to_s
         if from.empty?
-          date = @doc.at(
+          date = @doc.at_xpath(
             "//span[@itemprop='releaseDate']",
             "//ul[@id='stages']/li[contains(@class,'active')]/ul/li[@class='active']/a/span[@class='stage-date']",
           )
@@ -585,7 +585,7 @@ module Relaton
       # @return [Relaton::Bib::Contributor, nil]
       #
       def fetch_editorialgroup_contributor # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
-        wg = @doc.at(
+        wg = @doc.at_xpath(
           "//div[contains(., 'Technical Committe')]" \
           "/following-sibling::span/a",
         )

data/lib/relaton/itu/bibliography.rb

@@ -5,12 +5,28 @@ module Relaton
     module Bibliography
       extend self
 
-      # @param refid [Relaton::Itu::Pubid, String] a document reference
+      # Supplements that print the date of the recommendation they amend
+      # first, "ITU-T G.989.2 (2014) Amd. 1 (04/2016)": their edition is the
+      # base's.
+      AMENDING = [
+        ::Pubid::Itu::Identifiers::Amendment, ::Pubid::Itu::Identifiers::Corrigendum,
+        ::Pubid::Itu::Identifiers::Errata, ::Pubid::Itu::Identifiers::Addendum,
+      ].freeze
+
+      # Parts of a recommendation that print its date first when it has one,
+      # "ITU-T Z.100 (06/2021) Annex F1", and their own date otherwise.
+      ANNEXED = [
+        ::Pubid::Itu::Identifiers::AnnexOfRecommendation,
+        ::Pubid::Itu::Identifiers::AppendixOfRecommendation,
+      ].freeze
+
+      # @param refid [Pubid::Itu::Identifier, String] a document reference; a
+      #   String that Pubid::Itu cannot parse raises Pubid::Errors::ParseError
       # @return [Relaton::Itu::HitCollection]
       def search(refid)
         if refid.is_a? String
           warn_incorrect_ref(refid)
-          refid = Pubid.parse refid
+          refid = ::Pubid::Itu.parse refid
         end
         HitCollection.new(refid).tap(&:search)
       end
@@ -21,19 +37,60 @@ module Relaton
       # @return [Relaton::Bib::ItemData, nil]
       def get(code, year = nil, opts = {})
         warn_incorrect_ref(code)
-        refid = Pubid.parse code
-        refid.year ||= year
+        refid = with_year ::Pubid::Itu.parse(code), year
 
         ret = itubib_get1(refid)
         return nil if ret.nil?
 
-        ret = ret.to_most_recent_reference unless refid.year || opts[:keep_year]
+        unless edition_year(refid) || opts[:keep_year]
+          ret = ret.to_most_recent_reference
+        end
         ret = ret.to_all_parts if opts[:all_parts]
         ret
       end
 
+      # The year a reference selects its hits by. It is read the way a hit
+      # code's first date is: for an amending supplement, it is the year of
+      # the recommendation it amends. HitCollection uses it too.
+      #
+      # @param refid [Pubid::Itu::Identifier]
+      # @return [String, nil]
+      def edition_year(refid)
+        return refid.base&.year if amending?(refid)
+        return refid.base&.year || refid.year if annexed?(refid)
+
+        refid.year
+      end
+
+      # The identifier a reference is a part of: the recommendation an
+      # amending supplement, an annex or an appendix belongs to; else itself.
+      # A series or recommendation supplement is a document of its own.
+      #
+      # @param refid [Pubid::Itu::Identifier]
+      # @return [Pubid::Itu::Identifier]
+      def document(refid)
+        amending?(refid) || annexed?(refid) ? refid.base : refid
+      end
+
       private
 
+      # @param refid [Pubid::Itu::Identifier]
+      # @param year [String, nil] the year passed beside the reference
+      # @return [Pubid::Itu::Identifier] refid, dated with year when it has
+      #   no year of its own
+      def with_year(refid, year)
+        return refid if year.nil? || edition_year(refid)
+
+        hash = refid.to_hash
+        dated = amending?(refid) ? hash["base"] : hash
+        dated["year"] = year.to_s
+        ::Pubid::Itu::Identifier.from_hash hash
+      end
+
+      def amending?(refid) = AMENDING.any? { |klass| refid.is_a? klass }
+
+      def annexed?(refid) = ANNEXED.any? { |klass| refid.is_a? klass }
+
       def warn_incorrect_ref(ref)
         if ref =~ /(ITU[\s-]T\s\w)\.(Suppl\.|Annex)\s?(\w?\d+)/
           correct_ref = "#{$~[1]} #{$~[2]} #{$~[3]}"
@@ -45,30 +102,59 @@ module Relaton
         Util.info "Not found.", key: refid.to_s
         # one hit per edition means a year can repeat (a recommendation and its
         # amendment); the requested year itself is not a "different year" hint
-        years = missed_years.uniq.reject { |y| y == refid.year }
+        year = edition_year(refid)
+        years = missed_years.uniq.reject { |y| y == year }
         if years.any?
           plural = years.size > 1 ? "s" : ""
-          Util.info "There was no match for `#{refid.year}` year, though there were matches " \
-                    "found for `#{years.join('`, `')}` year#{plural}.", key: refid.to_s
+          Util.info "There was no match for `#{year}` year, though there " \
+                    "were matches found for `#{years.join('`, `')}` " \
+                    "year#{plural}.", key: refid.to_s
         end
         nil
       end
 
+      # Keep the hits that are the referenced document, in any edition:
+      # #isobib_results_filter picks the year afterwards. The date is ignored,
+      # and the version only when the reference names none. Everything else
+      # must be equal, so a supplement, or a Report for a Recommendation, never
+      # matches.
+      #
+      # Not `===`: it reads a missing `subseries` as "any value", so
+      # `ITU-T H.264` would keep `ITU-T H.264.1`.
       def search_filter(refid)
-        result = search(refid)
-        result.select do |i|
+        ignore = %i[year month day]
+        ignore << :version unless stated_version?(refid)
+        search(refid).select do |i|
           next true unless i.hit[:code]
 
-          pubid = Pubid.parse i.hit[:code]
-          refid.===(pubid, [:year])
+          hit = ::Pubid::Itu.parse(i.hit[:code])
+          refid.matches?(hit, ignore: ignore) && same_own_date?(refid, hit)
+        rescue ::Pubid::Errors::ParseError
+          false # a live hit ITU spells in a form nothing can identify
         end
       end
 
+      # A version on the reference, or on the recommendation it amends.
+      def stated_version?(refid)
+        id = document(refid)
+        id.respond_to?(:version) && !id.version.nil?
+      end
+
+      # `ignore:` drops the date at every level, but an amending supplement's
+      # own date tells two "Amd. 1" to different editions apart.
+      def same_own_date?(refid, hit)
+        return true unless amending?(refid)
+
+        own = refid.to_hash.slice("year", "month", "day")
+        own.all? { |key, value| hit.to_hash[key] == value }
+      end
+
       def isobib_results_filter(result, refid)
         missed_years = []
+        year = edition_year(refid)
         result.each do |r|
           /\((?:\d{2}\/)?(?<pyear>\d{4})\)/ =~ r.hit[:code]
-          if !refid.year || refid.year == pyear
+          if !year || year == pyear
             ret = r.item
             return { ret: ret } if ret
           end

data/lib/relaton/itu/data_crawler_r.rb

@@ -564,7 +564,7 @@ module Relaton
       # the first cell, the title in the second.
       #
       # @param url [String]
-      # @return [Array<Array(String, Nokogiri::XML::Element, Nokogiri::XML::NodeSet)>]
+      # @return [Array<Array(String, Moxml::Element, Moxml::NodeSet)>]
       def rows(url)
         get(url).search("//a[contains(@href,'parent=R-')]").filter_map do |a|
           id = a[:href].to_s[PARENT_RE, :id]
@@ -576,7 +576,7 @@ module Relaton
       # The title cell carries an optional red annotation ("Note - Suppressed on
       # …"), which is not part of the title.
       #
-      # @param cell [Nokogiri::XML::Element, nil]
+      # @param cell [Moxml::Element, nil]
       # @return [String]
       def title_text(cell)
         return "" unless cell

data/lib/relaton/itu/hit_collection.rb

@@ -24,21 +24,21 @@ module Relaton
       RECEDITIONS_URL = "#{DOMAIN}/mws/api/recommendations/getRecEditions?idrec=%<idrec>s&lang=en".freeze
       HANDLE_URL = "http://handle.itu.int/11.1002/1000/%<idrec>s-en".freeze
 
+      # `ref` is a Pubid::Itu identifier; its class and sector pick the route.
       def search
-        case ref.to_ref
-        # `ref` is a parsed Pubid, so only the abbreviated `OB.N` form survives
-        # here — the local grammar reduces "Operational Bulletin 1096" to the
-        # code "Operational" and drops the year, which no route can resolve.
-        when %r{^ITU-R\sRR}, %r{\bOB\.} then request_publication
-        when /^ITU-T/ then request_recommendation
-        # A Report reference renders "Report ITU-R …" (the discriminator leads,
-        # as ITU cites it), so it no longer starts with the sector — but it is
-        # still served from the index like any other ITU-R document.
-        when /^ITU-R\s/, /^Report\sITU-R\s/ then request_document
+        if publication?(ref) then request_publication
+        # an annex to a bulletin has no landing page, and it is not a
+        # recommendation either
+        elsif publication?(ref.root) then @array = []
+        else
+          case sector
+          when "T" then request_recommendation
+          when "R" then request_document
+          end
         end
       rescue Mechanize::ResponseCodeError, SocketError, Timeout::Error, Errno::ECONNRESET,
               EOFError, Net::ProtocolError, OpenSSL::SSL::SSLError => e
-        raise Relaton::RequestError, "Could not access #{ref.to_ref}: #{e.message}"
+        raise Relaton::RequestError, "Could not access #{ref}: #{e.message}"
       end
 
       def agent
@@ -70,6 +70,21 @@ module Relaton
 
       private
 
+      # Radio Regulations and Operational Bulletins live on www.itu.int/pub,
+      # not in the dataset.
+      def publication?(id)
+        id.is_a?(::Pubid::Itu::Identifiers::RadioRegulations) ||
+          id.is_a?(::Pubid::Itu::Identifiers::SpecialPublication)
+      end
+
+      # An annex or an appendix carries its sector on the recommendation it
+      # belongs to.
+      #
+      # @return [String, nil] "T" or "R"
+      def sector
+        (ref.sector || ref.root.sector)&.to_s
+      end
+
       # ITU-T Recommendations and supplements. The index is preferred (offline,
       # every edition, no scraping); the live path covers what the dataset does
       # not have — Implementers' Guides, the `(V##)`/`Annex` forms the data
@@ -86,9 +101,8 @@ module Relaton
 
       # @return [Array<Relaton::Itu::Hit>] newest edition first
       def index_hits
-        pubid = index_pubid_ref
-        return [] unless pubid
-
+        # Every edition is a candidate; Bibliography selects the year.
+        pubid = ref.exclude(:year, :month, :day)
         Util.info "Fetching from Relaton repository ...", key: ref.to_s
         index.search(pubid) { |i| index_match?(pubid, i[:id]) }
           .map { |r| index_hit r }.sort_by { |h| edition_key h.hit[:code] }.reverse
@@ -112,7 +126,10 @@ module Relaton
       def year_satisfied?(hits)
         return false if hits.empty?
 
-        ref.year.nil? || hits.any? { |h| edition_key(h.hit[:code]).first == ref.year.to_i }
+        year = Bibliography.edition_year(ref)
+        return true if year.nil?
+
+        hits.any? { |h| edition_key(h.hit[:code]).first == year.to_i }
       end
 
       # The edition date of a rendered docidentifier, as `[year, month]`. Uses
@@ -135,14 +152,13 @@ module Relaton
 
       def request_document # rubocop:todo Metrics/MethodLength, Metrics/AbcSize
         Util.info "Fetching from Relaton repository ...", key: ref.to_s
-        # Pass the reference's parsed pubid (not a String) so Relaton::Index can
+        # Pass the reference as a pubid (not a String) so Relaton::Index can
         # narrow candidates by document number before the block; `index_match?`
         # then matches every edition when the reference omits the part and the
         # exact edition when it names one. Rows are Pubid::Itu objects (not
         # Comparable), so rank by the numeric edition in `code.parts` (`["3"]` → 3)
         # to return the latest — index order isn't by edition.
-        pubid = pubid_ref
-        row = index.search(pubid) { |i| index_match?(pubid, i[:id]) }
+        row = index.search(ref) { |i| index_match?(ref, i[:id]) }
           .max_by { |i| i[:id].code&.parts&.last.to_i }
         return unless row
 
@@ -155,30 +171,6 @@ module Relaton
         @array = [hit]
       end
 
-      # Parse the reference into a Pubid::Itu identifier for the index lookup. A
-      # ref pubid can't parse raises (a `Pubid`/`Parslet` error) and propagates to
-      # the caller (relaton-cli / API callers rescue it), mirroring the ETSI
-      # flavor — the consumer no longer degrades to a raw-string substring search.
-      #
-      # @return [::Pubid::Itu::Identifier]
-      def pubid_ref
-        ::Pubid::Itu.parse ref.to_ref
-      end
-
-      # The reference as a pubid for an ITU-T index lookup, stripped of its
-      # edition date (every edition is a candidate; Bibliography selects the
-      # year). Unlike `#pubid_ref` this **rescues** an unparseable reference: the
-      # ITU-T path has a live fallback to degrade to, and a malformed reference
-      # (e.g. `ITU-T G.Suppl.47`) should warn and report "Not found", not raise.
-      #
-      # @return [::Pubid::Itu::Identifier, nil]
-      def index_pubid_ref
-        undated = ref.dup.tap { |r| r.year = r.month = r.day = nil }
-        ::Pubid::Itu.parse undated.to_ref
-      rescue StandardError
-        nil
-      end
-
       # Does an index row's id match the reference? When the reference omits the
       # part/edition (a bare `ITU-R P.838`), every edition of that exact document
       # matches — "search all parts"; when it names a part (`ITU-R P.838-2`), only
@@ -227,9 +219,15 @@ module Relaton
         nil
       end
 
-      # @return [String] the `rec=` value for the rec.aspx lookup
+      # The `rec=` value for the rec.aspx lookup: the document without its
+      # sector, edition date or version — "H.264", "A Suppl. 2",
+      # "G.989 Suppl. 1". An amendment or an annex is looked up by the
+      # recommendation it belongs to.
+      #
+      # @return [String]
       def rec_query
-        ref.suppl ? "#{ref.code} Suppl. #{ref.suppl}" : ref.code
+        Bibliography.document(ref).exclude(:year, :month, :day, :version)
+          .to_s.delete_prefix("ITU-T ")
       end
 
       # @param idrec [String]
@@ -244,10 +242,14 @@ module Relaton
       # @param edition [Hash] a getRecEditions entry
       # @return [Relaton::Itu::Hit]
       def recommendation_hit(edition)
+        # The crawler's rec_name canonicalisation ("G 231" -> "G.231"): the
+        # code is parsed with Pubid::Itu, which rejects the space spelling.
+        require_relative "data_parser_t"
+        name = DataParserT.normalize_rec_name edition["rec_name"]
         # No `:ref` key: it drives Hit#gi_imp, which would route an Implementers'
         # Guide reference onto the getImplGuides endpoints.
         Hit.new({
-          code: "ITU-T #{edition['rec_name']}",
+          code: "ITU-T #{name}",
           title: edition["title"],
           url: format(HANDLE_URL, idrec: edition["idrec"]),
           type: "recommendation",
@@ -279,21 +281,31 @@ module Relaton
       end
 
       # The /pub identifier for a Radio Regulation or an Operational Bulletin
-      # (`OB.1096` → `T-SP-OB.1096-2016`). nil — no hit — rather than a bogus URL
-      # when the bulletin reference carries no number.
+      # (`OB.1096` → `T-SP-OB.1096-2016`). nil — no hit — for any other special
+      # publication, which has no derivable landing page.
       #
       # @return [String, nil]
       def publication_id
-        return "R-REG-RR-#{ref.year}" if ref.code == "RR"
+        return "R-REG-RR-#{ref.year}" if radio_regulations?
 
-        num = ref.code[/\d+/]
-        "T-SP-OB.#{num}-#{ref.year}" if num
+        "T-SP-OB.#{ref.number}-#{ref.year}" if bulletin?
       end
 
-      # @return [String] the docidentifier-friendly code (year only, no month)
+      # The docidentifier (year only, no month or day). A bulletin keeps its
+      # sector even when the reference is the sector-less `ITU OB No. 1096`.
+      #
+      # @return [String]
       def publication_code
-        "#{ref.prefix}-#{ref.sector} #{ref.code} (#{ref.year})"
+        return "ITU-R RR (#{ref.year})" if radio_regulations?
+
+        "ITU-T OB.#{ref.number} (#{ref.year})"
       end
+
+      def radio_regulations?
+        ref.is_a? ::Pubid::Itu::Identifiers::RadioRegulations
+      end
+
+      def bulletin? = ref.series.to_s == "OB"
     end
   end
 end

data/lib/relaton/itu/processor.rb

@@ -6,6 +6,7 @@ module Relaton
       def initialize
         @short = :relaton_itu
         @prefix = "ITU"
+        @pubid_identifier = :Itu # Db cache key
         @defaultprefix = %r{^ITU\s}
         @idtype = "ITU"
         @datasets = %w[itu-r itu-t]

data/lib/relaton/itu/scraper.rb

@@ -112,13 +112,23 @@ module Relaton
           type = "ISO"
           text.match(/[^(]+/).to_s.strip.squeeze(" ")
         else
-          pubid = Pubid.parse(text)
-          type = pubid.prefix
-          pubid.to_s
+          type = "ITU"
+          canonical_id text
         end => id
         Docidentifier.new(type: type, content: id, primary: true)
       end
 
+      # The pubid spelling of an ITU code. The code is data from ITU, so a
+      # form pubid cannot identify is kept as ITU wrote it.
+      #
+      # @param text [String]
+      # @return [String]
+      def canonical_id(text)
+        ::Pubid::Itu.parse(text).to_s
+      rescue ::Pubid::Errors::ParseError
+        text
+      end
+
       # @return [Array<Relaton::Bib::Contributor>]
       def fetch_contributors
         contribs = fetch_publisher_contributors

data/lib/relaton/itu.rb

@@ -1,5 +1,4 @@
 require "mechanize"
-require "parslet"
 require "pubid"
 require "digest/md5"
 require "relaton/index"
@@ -11,7 +10,6 @@ require_relative "itu/item_data"
 require_relative "itu/model/item"
 require_relative "itu/model/bibitem"
 require_relative "itu/model/bibdata"
-require_relative "itu/pubid"
 require_relative "itu/scraper"
 require_relative "itu/hit_collection"
 require_relative "itu/bibliography"

data/lib/relaton/jis/data_fetcher.rb

@@ -114,12 +114,12 @@ module Relaton
       end
 
       def parse_offset(resp) # rubocop:disable Metrics/AbcSize
-        if resp.at('//*[@id="btnPaging"]') # first page
+        if resp.at_xpath('//*[@id="btnPaging"]') # first page
           xpath = '//script[contains(.,"var count =")]'
-          @count = resp.at(xpath).text.match(/var count = (\d+);/)[1]
-          resp.at("//*[@id='offset']")[:value].to_i
+          @count = resp.at_xpath(xpath).text.match(/var count = (\d+);/)[1]
+          resp.at_xpath("//*[@id='offset']")[:value].to_i
         else
-          script = resp.at("//script").text
+          script = resp.at_xpath("//script").text
           script.match(/\("offset"\)\.value = '(\d+)'/)[1].to_i
         end
       end

data/lib/relaton/jis/processor.rb

@@ -10,6 +10,7 @@ module Relaton
       def initialize # rubocop:disable Lint/MissingSuper
         @short = :relaton_jis
         @prefix = "JIS"
+        @pubid_identifier = :Jis # Db cache key
         @defaultprefix = %r{^(JIS|TR)\s}
         @idtype = "JIS"
         @datasets = %w[jis-webdesk]