relaton 3.0.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/itu/family_cache.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Itu
+    #
+    # Cross-row cache for the ITU-T detail endpoints that answer the same thing
+    # for every edition of one recommendation.
+    #
+    # `searchRecs` runs with `main_edition_flag=0`, so it returns one row **per
+    # edition** — ~16k rows over ~7.7k recommendations. Enrichment then costs
+    # four www.itu.int requests per row, and three of them are keyed by `idrec`
+    # but answer for the whole recommendation. Verified from
+    # `spec/itu/vcr_cassettes/itu_t_h_264.yml`: `getRecEditions?idrec=16818` and
+    # `?idrec=14659` return **byte-identical** 21-row payloads. H.264's 21
+    # editions therefore issue 21 identical requests, and the corpus average is
+    # ~2.1 editions per recommendation.
+    #
+    # The cache is created per crawl by `DataFetcher#fetch_recommendations` and
+    # threaded down as a parameter — never global state, and never reachable
+    # from the live `Bibliography.get` path, which gets {NullCache} instead.
+    #
+    # Two locks, and the split matters. `@mutex` guards only the entry *table*;
+    # each entry carries its own lock, which the block (an HTTP request) is run
+    # under. Holding `@mutex` across the request would serialize all eight
+    # workers onto one connection's latency and collapse the pool to one thread;
+    # the per-entry lock is also what makes two workers racing on two editions
+    # of one recommendation issue **one** request rather than two.
+    #
+    class FamilyCache
+      # Entries resident at once. Bounded because an unbounded cache over ~7.7k
+      # recommendations, each holding an editions list and a supplements list,
+      # runs to hundreds of MB on a CI runner alongside eight Mechanize agents.
+      # With rows enqueued family-by-family only a handful are ever live, so the
+      # cap costs no hit rate.
+      DEFAULT_MAX_ENTRIES = 512
+
+      Entry = Struct.new(:mutex, :value, :computed)
+
+      def initialize(max_entries: DEFAULT_MAX_ENTRIES)
+        @max = [max_entries.to_i, 1].max
+        @entries = {}
+        @mutex = Mutex.new
+        @hits = 0
+        @misses = 0
+        @evictions = 0
+      end
+
+      # Value for `key`, computing it under that key's own lock when absent.
+      #
+      # A block that raises leaves the entry uncached, so a transient failure is
+      # retried by the next caller instead of being served to the whole family
+      # as a permanent nil.
+      #
+      # @param key [Object]
+      # @return [Object]
+      def fetch(key)
+        entry = @mutex.synchronize { touch key }
+        entry.mutex.synchronize do
+          if entry.computed
+            @mutex.synchronize { @hits += 1 }
+            next entry.value
+          end
+
+          value = yield
+          entry.value = value
+          entry.computed = true
+          @mutex.synchronize do
+            @misses += 1
+            # Re-publish. The table lock is released between #touch and the
+            # entry lock above, and in that window this entry is unlocked and so
+            # a valid eviction candidate for another thread's #touch. Nothing
+            # wrong is ever served — this thread holds the Entry directly — but
+            # without this the write is lost and the next caller pays for the
+            # request again. `unless key?` so a replacement entry another thread
+            # has since created is not discarded.
+            @entries[key] = entry unless @entries.key? key
+          end
+          value
+        end
+      end
+
+      # Publish an already-computed value under every other key it also answers.
+      #
+      # This is what makes one request warm a whole family: a `getRecEditions`
+      # response names every sibling `idrec`, and by the endpoint's own testimony
+      # the payload for each of them is the same one. An entry whose lock is held
+      # is skipped — it is mid-fetch and will land on this value anyway.
+      #
+      # @param keys [Array<Object>]
+      # @param value [Object]
+      # @return [Object] value
+      def warm(keys, value)
+        keys.each do |key|
+          entry = @mutex.synchronize { touch key }
+          next unless entry.mutex.try_lock
+
+          begin
+            next if entry.computed
+
+            entry.value = value
+            entry.computed = true
+          ensure
+            entry.mutex.unlock
+          end
+        end
+        value
+      end
+
+      # @return [Boolean] whether this cache actually caches. Callers use it to
+      #   skip building a key whose computation would itself cost a request.
+      def caching?
+        true
+      end
+
+      # @return [Hash] for the run summary
+      def stats
+        @mutex.synchronize do
+          { hits: @hits, misses: @misses, entries: @entries.size, evictions: @evictions }
+        end
+      end
+
+      private
+
+      # Fetch-or-create `key` and mark it most recently used. Callers hold
+      # @mutex. Eviction skips an entry currently being computed, so a worker
+      # can never lose the entry it is holding.
+      def touch(key)
+        if @entries.key? key
+          entry = @entries.delete key
+          return @entries[key] = entry
+        end
+
+        evict while @entries.size >= @max
+        @entries[key] = Entry.new(Mutex.new, nil, false)
+      end
+
+      def evict
+        victim = @entries.each_key.find { |k| !@entries[k].mutex.locked? }
+        # Everything resident is in flight: grow rather than block a worker.
+        return @max = @entries.size + 1 unless victim
+
+        @entries.delete victim
+        @evictions += 1
+      end
+    end
+
+    #
+    # The no-op cache: `#fetch` is a bare yield.
+    #
+    # What the live `Bibliography.get` path gets, and the default everywhere, so
+    # runtime behaviour and runtime request count are byte-identical to what
+    # they were before the cache existed and no state survives a lookup. Only
+    # `DataParserT`, walking one corpus of 16k rows, injects a real one.
+    #
+    class NullCache
+      def self.instance
+        @instance ||= new
+      end
+
+      def fetch(_key)
+        yield
+      end
+
+      def warm(_keys, value)
+        value
+      end
+
+      def caching?
+        false
+      end
+
+      def stats
+        { hits: 0, misses: 0, entries: 0, evictions: 0 }
+      end
+    end
+  end
+end

data/lib/relaton/itu/governor.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "mechanize"
+require_relative "../core/governor"
+
+module Relaton
+  module Itu
+    #
+    # ITU binding of {Relaton::Core::Governor}, for the ITU-R crawl.
+    #
+    # ITU does not signal throttling with a distinct exception class the way
+    # api.w3.org does, so membership is decided by status *code* and by one
+    # marker module rather than by an error hierarchy. Measured over a
+    # 49-minute full run, the F5 WAF in front of www.itu.int answers `/rec`
+    # with **HTTP 503** and `/pub` with a **302 to notfound.aspx** — a soft
+    # block Mechanize follows to a perfectly good 200, indistinguishable from
+    # "no such document" unless the crawler marks it (see
+    # `DataCrawlerR::SoftBlock`, which includes {SoftBlock} for exactly this).
+    #
+    # Why a governor at all, when `DataCrawlerR#get` already retries: a rate
+    # limit is a *pool* event. One worker's 503 must pause every other worker,
+    # for a window that outlasts the ban, instead of four workers each running
+    # their own 5/10/15 s ladder and keeping the limiter engaged. That is the
+    # failure the W3C crawl spent 4 h 56 m in before this class existed.
+    #
+    class Governor < Relaton::Core::Governor
+      # Mixed into the crawler's soft-block error. A module rather than a class
+      # so the exception can stay a `Mechanize::ResponseCodeError` subclass —
+      # which is what keeps `#get`'s existing rescue list unchanged — while
+      # still being distinguishable from the 404 it otherwise looks like.
+      module SoftBlock; end
+
+      # Response codes that mean "the WAF is pushing back", as opposed to "that
+      # document is not there" or "that record is broken".
+      #
+      # **500 is deliberately absent.** Same call the W3C governor makes: a
+      # persistent 5xx on one resource is a broken record, and routing it here
+      # would open a pool-wide cooldown for every bad page in the corpus. Only
+      # 503 (measured, on /rec) and the gateway codes, which are infrastructure
+      # rather than content, get the pool treatment.
+      THROTTLE_CODES = %w[429 502 503 504].freeze
+
+      # RELATON_ITU_THROTTLE_BASE / _MAX / _GIVEUP.
+      ENV_PREFIX = "RELATON_ITU"
+
+      # @param error [StandardError]
+      # @return [Boolean]
+      def self.throttle?(error)
+        return true if error.is_a? SoftBlock
+        return false unless error.is_a? ::Mechanize::ResponseCodeError
+
+        THROTTLE_CODES.include? error.response_code.to_s
+      end
+    end
+  end
+end

data/lib/relaton/itu/hit.rb

@@ -6,10 +6,16 @@ module Relaton
     class Hit < Relaton::Core::Hit
       attr_writer :item
 
-      # Parse page.
-      # @return [Relaton::Itu::ItemData]
+      # The document itself. An index-derived hit (`:file`) is loaded from the
+      # data repository on demand — a reference matching many editions builds a
+      # hit per edition, and only the one Bibliography selects is downloaded.
+      # Everything else is scraped from www.itu.int.
+      #
+      # @return [Relaton::Itu::ItemData, nil]
       def item
-        @item ||= Scraper.parse_page self, imp: gi_imp
+        @item ||= if hit[:file] then hit_collection.fetch_item(hit[:url])
+                  else Scraper.parse_page self, imp: gi_imp
+                  end
       end
 
       private

data/lib/relaton/itu/hit_collection.rb

@@ -1,20 +1,40 @@
 # frozen_string_literal: true
 
+require "cgi"
+require "json"
 require_relative "hit"
 
 module Relaton
   module Itu
     # Page of hit collection.
+    #
+    # Reference discovery has three sources, because ITU decommissioned the
+    # `net4/.../GlobalSearch/RunSearch` endpoint that used to serve all of them:
+    #
+    # - Recommendations (`ITU-T …`) and ITU-R publications come from the combined
+    #   `relaton-data-itu` `index-v2`; an ITU-T reference the index can't serve
+    #   falls back to the live `rec.aspx` + `getRecEditions` pair.
+    # - Radio Regulations (`ITU-R RR …`) and Operational Bulletins (`… OB.N …`)
+    #   are not in the dataset; their landing page URL is derivable from the
+    #   reference, so they are resolved directly against `www.itu.int/pub/…`.
     class HitCollection < Relaton::Core::HitCollection
       DOMAIN = "https://www.itu.int"
-      GH_ITU_R = "https://raw.githubusercontent.com/relaton/relaton-data-itu-r/refs/heads/v2/"
+      GH_ITU = "https://raw.githubusercontent.com/relaton/relaton-data-itu/refs/heads/main/"
+      REC_URL = "#{DOMAIN}/ITU-T/recommendations/rec.aspx?rec=%<rec>s&lang=en".freeze
+      RECEDITIONS_URL = "#{DOMAIN}/mws/api/recommendations/getRecEditions?idrec=%<idrec>s&lang=en".freeze
+      HANDLE_URL = "http://handle.itu.int/11.1002/1000/%<idrec>s-en".freeze
 
       def search
         case ref.to_ref
-        when /^(ITU-T|ITU-R\sRR)/
-          request_search
-        when /^ITU-R\s/
-          request_document
+        # `ref` is a parsed Pubid, so only the abbreviated `OB.N` form survives
+        # here — the local grammar reduces "Operational Bulletin 1096" to the
+        # code "Operational" and drops the year, which no route can resolve.
+        when %r{^ITU-R\sRR}, %r{\bOB\.} then request_publication
+        when /^ITU-T/ then request_recommendation
+        # A Report reference renders "Report ITU-R …" (the discriminator leads,
+        # as ITU cites it), so it no longer starts with the sector — but it is
+        # still served from the index like any other ITU-R document.
+        when /^ITU-R\s/, /^Report\sITU-R\s/ then request_document
         end
       rescue Mechanize::ResponseCodeError, SocketError, Timeout::Error, Errno::ECONNRESET,
               EOFError, Net::ProtocolError, OpenSSL::SSL::SSLError => e
@@ -25,102 +45,254 @@ module Relaton
         @agent ||= Mechanize.new.tap { |agent| agent.user_agent_alias = "Mac Safari" }
       end
 
+      # Fetch and parse one document from the data repository. Public because
+      # `Hit#item` calls it to load an index-derived hit lazily — a reference
+      # that matches many editions must not download them all.
+      #
+      # @param url [String] raw URL of the document's YAML
+      # @return [Relaton::Itu::ItemData, nil] nil when the file is missing
+      def fetch_item(url)
+        resp = agent.get url
+        return if resp.code == "404"
+
+        Item.from_yaml(resp.body).tap { |i| i.fetched = Date.today.to_s }
+      rescue Mechanize::ResponseCodeError, SocketError, Timeout::Error, Errno::ECONNRESET,
+             EOFError, Net::ProtocolError, OpenSSL::SSL::SSLError => e
+        # Mechanize *raises* on 404 rather than returning the response, so an
+        # index row whose data file has gone missing is caught here —
+        # Bibliography then moves on to the next edition instead of failing.
+        return if e.is_a?(Mechanize::ResponseCodeError) && e.response_code == "404"
+
+        # `#search` wraps its own failures, but a lazy hit is loaded later, from
+        # `Hit#item` — outside that rescue.
+        raise Relaton::RequestError, "Could not access #{url}: #{e.message}"
+      end
+
       private
 
-      def request_search
-        Util.info "Fetching from www.itu.int ...", key: ref.to_s
-        url = "#{DOMAIN}/net4/ITU-T/search/GlobalSearch/RunSearch"
-        data = { json: params.to_json }
-        resp = agent.post url, data
-        @array = hits JSON.parse(resp.body)
+      # ITU-T Recommendations and supplements. The index is preferred (offline,
+      # every edition, no scraping); the live path covers what the dataset does
+      # not have — Implementers' Guides, the `(V##)`/`Annex` forms the data
+      # repo's crawler leaves unindexed, and editions approved since its last
+      # crawl. Indexed editions are kept when the live path finds nothing, so a
+      # wrong-year reference still reports the years that do exist.
+      def request_recommendation
+        hits = index_hits
+        return @array = hits if year_satisfied? hits
+
+        live = live_hits
+        @array = live.empty? ? hits : live
+      end
+
+      # @return [Array<Relaton::Itu::Hit>] newest edition first
+      def index_hits
+        pubid = index_pubid_ref
+        return [] unless pubid
+
+        Util.info "Fetching from Relaton repository ...", key: ref.to_s
+        index.search(pubid) { |i| index_match?(pubid, i[:id]) }
+          .map { |r| index_hit r }.sort_by { |h| edition_key h.hit[:code] }.reverse
+      end
+
+      # @param row [Hash] an index row
+      # @return [Relaton::Itu::Hit]
+      def index_hit(row)
+        # `row[:id].to_s` renders the dated docidentifier the data record itself
+        # carries ("ITU-T L.163 (11/2018)"), which is what Bibliography's year
+        # filtering parses — so no separate edition bookkeeping is needed.
+        Hit.new({ code: row[:id].to_s, url: "#{GH_ITU}#{row[:file]}", file: row[:file] }, self)
+      end
+
+      # Can the index answer the reference? A year-less reference is satisfied by
+      # any hit; a dated one needs the year to be present, otherwise a newer (or
+      # unindexed) edition has to come from www.itu.int.
+      #
+      # @param hits [Array<Relaton::Itu::Hit>]
+      # @return [Boolean]
+      def year_satisfied?(hits)
+        return false if hits.empty?
+
+        ref.year.nil? || hits.any? { |h| edition_key(h.hit[:code]).first == ref.year.to_i }
+      end
+
+      # The edition date of a rendered docidentifier, as `[year, month]`. Uses
+      # the **first** date in the string, mirroring Bibliography's year filter:
+      # an amendment ("ITU-T G.989.2 (2014) Amd 1 (04/2016)") is identified by
+      # the year of the recommendation it amends.
+      #
+      # @param code [String, nil]
+      # @return [Array<Integer>]
+      def edition_key(code)
+        month, year = code.to_s.match(%r{\((?:(\d{2})/)?(\d{4})\)})&.captures
+        [year.to_i, month.to_i]
+      end
+
+      def index
+        Relaton::Index.find_or_create :itu, url: "#{GH_ITU}#{INDEXFILE}.zip",
+                                            file: "#{INDEXFILE}.yaml",
+                                            pubid_class: ::Pubid::Itu::Identifier
       end
 
       def request_document # rubocop:todo Metrics/MethodLength, Metrics/AbcSize
         Util.info "Fetching from Relaton repository ...", key: ref.to_s
-        index = Relaton::Index.find_or_create :itu, url: "#{GH_ITU_R}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml"
-        row = index.search(ref.to_ref).max_by { |i| i[:id] }
+        # Pass the reference's parsed pubid (not a String) so Relaton::Index can
+        # narrow candidates by document number before the block; `index_match?`
+        # then matches every edition when the reference omits the part and the
+        # exact edition when it names one. Rows are Pubid::Itu objects (not
+        # Comparable), so rank by the numeric edition in `code.parts` (`["3"]` → 3)
+        # to return the latest — index order isn't by edition.
+        pubid = pubid_ref
+        row = index.search(pubid) { |i| index_match?(pubid, i[:id]) }
+          .max_by { |i| i[:id].code&.parts&.last.to_i }
         return unless row
 
-        url = "#{GH_ITU_R}#{row[:file]}"
-        resp = agent.get url
-        return if resp.code == "404"
+        url = "#{GH_ITU}#{row[:file]}"
+        item = fetch_item url
+        return unless item
 
-        item = Item.from_yaml(resp.body).tap { |i| i.fetched = Date.today.to_s }
         hit = Hit.new({ url: url, ref: ref }, self)
         hit.item = item
         @array = [hit]
       end
 
-      # @return [String]
-      def group
-        @group ||= case ref.to_ref
-                   when %r{OB|Operational Bulletin}, %r{^ITU-R\sRR}
-                     "Publications"
-                   when %r{^ITU-T} then "Recommendations"
-                   end
-      end
-
-      # @return [Hash]
-      def params # rubocop:disable Metrics/MethodLength
-        input = ref.dup
-        input.year = nil
-        {
-          "Input" => input.to_s,
-          "Start" => 0,
-          "Rows" => 20,
-          "SortBy" => "RELEVANCE",
-          "ExactPhrase" => false,
-          "CollectionName" => "General",
-          "CollectionGroup" => group,
-          "Sector" => ref.to_ref.match(/(?<=^ITU-)\w/).to_s.downcase,
-          "Criterias" => [{
-            "Name" => "Search in",
-            "Criterias" => [
-              {
-                "Selected" => false,
-                "Value" => "",
-                "Label" => "Name",
-                "Target" => "/name_s",
-                "TypeName" => "CHECKBOX",
-                "GetCriteriaType" => 0,
-              },
-              {
-                "Selected" => false,
-                "Value" => "",
-                "Label" => "Short description",
-                "Target" => "/short_description_s",
-                "TypeName" => "CHECKBOX",
-                "GetCriteriaType" => 0,
-              },
-              {
-                "Selected" => false,
-                "Value" => "",
-                "Label" => "File content",
-                "Target" => "/file",
-                "TypeName" => "CHECKBOX",
-                "GetCriteriaType" => 0,
-              },
-            ],
-            "ShowCheckbox" => true,
-            "Selected" => false,
-          }],
-          "Topics" => "",
-          "ClientData" => {},
-          "Language" => "en",
-          "SearchType" => "All",
-        }
-      end
-
-      # @param data [Hash]
+      # Parse the reference into a Pubid::Itu identifier for the index lookup. A
+      # ref pubid can't parse raises (a `Pubid`/`Parslet` error) and propagates to
+      # the caller (relaton-cli / API callers rescue it), mirroring the ETSI
+      # flavor — the consumer no longer degrades to a raw-string substring search.
+      #
+      # @return [::Pubid::Itu::Identifier]
+      def pubid_ref
+        ::Pubid::Itu.parse ref.to_ref
+      end
+
+      # The reference as a pubid for an ITU-T index lookup, stripped of its
+      # edition date (every edition is a candidate; Bibliography selects the
+      # year). Unlike `#pubid_ref` this **rescues** an unparseable reference: the
+      # ITU-T path has a live fallback to degrade to, and a malformed reference
+      # (e.g. `ITU-T G.Suppl.47`) should warn and report "Not found", not raise.
+      #
+      # @return [::Pubid::Itu::Identifier, nil]
+      def index_pubid_ref
+        undated = ref.dup.tap { |r| r.year = r.month = r.day = nil }
+        ::Pubid::Itu.parse undated.to_ref
+      rescue StandardError
+        nil
+      end
+
+      # Does an index row's id match the reference? When the reference omits the
+      # part/edition (a bare `ITU-R P.838`), every edition of that exact document
+      # matches — "search all parts"; when it names a part (`ITU-R P.838-2`), only
+      # that edition matches. Delegates to pubid's structured `matches?`, ignoring
+      # `:parts` only when the reference omits the part — so a bare `ITU-R M.1`
+      # does not match `ITU-R M.10` (a different document number) and `-1` differs
+      # from `-10`, without the local `-`-separator anchor. Mirrors the ETSI
+      # flavor's `Bibliography#best_match`. Both ids are `Pubid::Itu` identifiers.
+      #
+      # `:year`/`:month`/`:version` are always ignored because ITU-T rows are one
+      # per **edition** (`ITU-T Z.100 (06/2021)`, `ITU-T H.264 (V14) (08/2021)`)
+      # while the reference is normalised to the undated form. They are separable
+      # trailing components, the same contract ETSI has (`omits: %i[version date]`);
+      # Bibliography then picks the requested year and version.
+      #
+      # @param pubid [::Pubid::Itu::Identifier] the reference
+      # @param id [::Pubid::Itu::Identifier] an index row's id
+      # @return [Boolean]
+      def index_match?(pubid, id)
+        ignore = %i[year month version]
+        ignore << :parts if pubid.code&.parts.to_a.empty?
+        pubid.matches?(id, ignore: ignore)
+      end
+
+      # Resolve an ITU-T Recommendation to its editions via the public rec.aspx
+      # page (which exposes the record's handle/idrec) and the getRecEditions
+      # API. One hit is built per edition, mirroring the multi-result shape the
+      # removed RunSearch endpoint returned, so year filtering keeps working.
+      #
       # @return [Array<Relaton::Itu::Hit>]
-      def hits(data)
-        data["results"].map do |h|
-          code  = h["Media"]["Name"]
-          title = h["Title"]
-          url   = "#{DOMAIN}#{h['Redirection']}"
-          type  = h["Collection"]["Group"].downcase[0...-1]
-          Hit.new({ code: code, title: title, url: url, type: type }, self)
-        end
+      def live_hits
+        Util.info "Fetching from www.itu.int ...", key: ref.to_s
+        idrec = fetch_idrec
+        return [] unless idrec
+
+        editions(idrec).map { |ed| recommendation_hit ed }
+      end
+
+      # @return [String, nil] the record's idrec, or nil when the code is unknown
+      def fetch_idrec
+        url = format(REC_URL, rec: CGI.escape(rec_query))
+        agent.get(url).body[%r{11\.1002/1000/(\d+)}, 1]
+      rescue Mechanize::ResponseCodeError => e
+        raise unless e.response_code == "404" # unknown code => treat as not found
+
+        nil
+      end
+
+      # @return [String] the `rec=` value for the rec.aspx lookup
+      def rec_query
+        ref.suppl ? "#{ref.code} Suppl. #{ref.suppl}" : ref.code
+      end
+
+      # @param idrec [String]
+      # @return [Array<Hash>] editions of the recommendation
+      def editions(idrec)
+        # a `null` body parses to nil, so `|| []` is not redundant
+        JSON.parse(agent.get(format(RECEDITIONS_URL, idrec: idrec)).body) || []
+      rescue JSON::ParserError
+        []
+      end
+
+      # @param edition [Hash] a getRecEditions entry
+      # @return [Relaton::Itu::Hit]
+      def recommendation_hit(edition)
+        # No `:ref` key: it drives Hit#gi_imp, which would route an Implementers'
+        # Guide reference onto the getImplGuides endpoints.
+        Hit.new({
+          code: "ITU-T #{edition['rec_name']}",
+          title: edition["title"],
+          url: format(HANDLE_URL, idrec: edition["idrec"]),
+          type: "recommendation",
+        }, self)
+      end
+
+      # Resolve an ITU-R Radio Regulation or Operational Bulletin to its stable
+      # /pub landing page, whose id is derivable from the reference. They are not
+      # part of the relaton-data-itu dataset.
+      def request_publication
+        Util.info "Fetching from www.itu.int ...", key: ref.to_s
+        id = ref.year && publication_id
+        return @array = [] unless id
+
+        url = "#{DOMAIN}/pub/#{id}"
+        page = fetch_publication_page url
+        return @array = [] if page.nil? || page.uri.to_s.match?(/notfound/i)
+
+        @array = [Hit.new({ code: publication_code, url: url, type: "publication" }, self)]
+      end
+
+      # @return [Mechanize::Page, nil] nil when the publication does not exist
+      def fetch_publication_page(url)
+        agent.get url
+      rescue Mechanize::ResponseCodeError => e
+        raise unless e.response_code == "404" # unknown publication => not found
+
+        nil
+      end
+
+      # The /pub identifier for a Radio Regulation or an Operational Bulletin
+      # (`OB.1096` → `T-SP-OB.1096-2016`). nil — no hit — rather than a bogus URL
+      # when the bulletin reference carries no number.
+      #
+      # @return [String, nil]
+      def publication_id
+        return "R-REG-RR-#{ref.year}" if ref.code == "RR"
+
+        num = ref.code[/\d+/]
+        "T-SP-OB.#{num}-#{ref.year}" if num
+      end
+
+      # @return [String] the docidentifier-friendly code (year only, no month)
+      def publication_code
+        "#{ref.prefix}-#{ref.sector} #{ref.code} (#{ref.year})"
       end
     end
   end