relaton-itu 2.1.1 → 2.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d26cbbb715a19961467253a86e1a90d1a238469d0f9011da6a4cd9d42ae67ea3
-  data.tar.gz: 3dd15dc14484d326f1b2ce57b817f9e56a902669934494da3f9df17cf3ac6837
+  metadata.gz: 6934a71bcfd52917864d6cf3af90fd07333ce87cc9c5d8cd045217d86a423b28
+  data.tar.gz: 0a1daa40935cb00409c1d40186aa63d7c25bd70866356a635fa52dc4ef512d87
 SHA512:
-  metadata.gz: cedaef1be6243a4c25eddac7b4b3418a11390485153cafb4792e0cec593a937c90e2f3895a6473dc46cdf67a7f5b34d6b47b301af7ccbb281bc11e060447071d
-  data.tar.gz: 7c86eee320f302f6d957fae54115bc5e06658ca30e44bdac89a9a44b5660a390272bbc817347656c1a1ebd33db5b9568cf28575b0113fe002cbe54bfaa41ae81
+  metadata.gz: 11d05f246d70e236f0995ec2a80abfef38c8085b602aa6bff5aa4c5033fdc5d4984cec8e9125348f9f31739903cf82e89d9d16819a4233e4855ece1488455366
+  data.tar.gz: 42cfd40edebcd128a3cc402a3c6e3c6fd4b2d7e1bad5a4a9d2403dd59b020cf729034eeaa864d915e69e1bf192b0bb7d2e90ce31465fa7db942dbaa608d3eb7f

data/CLAUDE.md

@@ -45,6 +45,28 @@ All model classes use `Lutaml::Model::Serializable` for XML/YAML serialization:
 - **`DataParserR`** — module that parses ITU-R JSON search API results into `ItemData` instances (sets `flavor: "itu"` on all parsed documents)
 - Sources: recommendations (JSON index), questions, reports, handbooks, resolutions (HTML indices)
 
+### Runtime lookup (`HitCollection#search`)
+
+`Bibliography.get`/`search` routes by reference in `HitCollection#search`:
+
+- **ITU-T Recommendations** (`request_recommendation`) — discovery is a two-stage HTTP flow against
+  `www.itu.int`. The `RunSearch` Deep Search endpoint was removed/WAF-blocked (issue #88), so
+  discovery now GETs the public `ITU-T/recommendations/rec.aspx?rec={code}` page, extracts the
+  record handle (`11.1002/1000/{idrec}`), and calls the `mws/api/recommendations/getRecEditions`
+  API to build one `Hit` per edition (each `hit[:url]` is a `handle.itu.int/.../{idrec}-en` URL so
+  `Scraper#idrec` re-parses it). Year/edition filtering stays in `Bibliography#search_filter` /
+  `#isobib_results_filter`. Full metadata is fetched by `RecommendationParser` (the `mws/api/*`
+  endpoints), unchanged.
+- **ITU-R RR & Operational Bulletins** (`request_publication`) — resolved to their predictable
+  `/pub/{pubid}` landing page (`R-REG-RR-{year}`, `T-SP-OB.{num}-{year}`) and scraped by
+  `RadioRegulationsParser`. A redirect to `notfound.aspx` (or a 404) yields no hit.
+- **ITU-R (other)** (`request_document`) — reads the pre-built `relaton-data-itu-r` GitHub dataset
+  via `Relaton::Index`; does not hit `www.itu.int`.
+
+Note: `DataFetcher` (the offline ITU-R harvester) still POSTs to the removed `RunSearch` endpoint,
+so a dataset *refresh* is currently broken — a follow-up should migrate it (and, longer term, ITU-T)
+to a relaton-data dataset.
+
 ### Processor
 
 `Relaton::Itu::Processor` extends `Relaton::Core::Processor` and is the entry point for the Relaton plugin system. Provides `get`, `fetch_data`, `from_xml`, `hash_to_bib`, `grammar_hash`, and `remove_index_file`.

data/lib/relaton/itu/hit_collection.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "cgi"
+require "json"
 require_relative "hit"
 
 module Relaton
@@ -8,11 +10,16 @@ module Relaton
     class HitCollection < Relaton::Core::HitCollection
       DOMAIN = "https://www.itu.int"
       GH_ITU_R = "https://raw.githubusercontent.com/relaton/relaton-data-itu-r/refs/heads/v2/"
+      REC_URL = "#{DOMAIN}/ITU-T/recommendations/rec.aspx?rec=%<rec>s&lang=en".freeze
+      RECEDITIONS_URL = "#{DOMAIN}/mws/api/recommendations/getRecEditions?idrec=%<idrec>s&lang=en".freeze
+      HANDLE_URL = "http://handle.itu.int/11.1002/1000/%<idrec>s-en".freeze
 
       def search
         case ref.to_ref
-        when /^(ITU-T|ITU-R\sRR)/
-          request_search
+        when %r{^ITU-R\sRR}, %r{\bOB\.|Operational Bulletin}
+          request_publication
+        when /^ITU-T/
+          request_recommendation
         when /^ITU-R\s/
           request_document
         end
@@ -27,12 +34,87 @@ module Relaton
 
       private
 
-      def request_search
+      # Resolve an ITU-T Recommendation to its editions via the public rec.aspx
+      # page (which exposes the record's handle/idrec) and the getRecEditions API.
+      # One hit is built per edition, mirroring the multi-result shape the old
+      # RunSearch endpoint used to return, so year filtering keeps working.
+      def request_recommendation
         Util.info "Fetching from www.itu.int ...", key: ref.to_s
-        url = "#{DOMAIN}/net4/ITU-T/search/GlobalSearch/RunSearch"
-        data = { json: params.to_json }
-        resp = agent.post url, data
-        @array = hits JSON.parse(resp.body)
+        idrec = fetch_idrec
+        return @array = [] unless idrec
+
+        @array = editions(idrec).map { |ed| recommendation_hit(ed) }
+      end
+
+      # @return [String, nil] the record's idrec, or nil when the code is unknown
+      def fetch_idrec
+        url = format(REC_URL, rec: CGI.escape(rec_query))
+        agent.get(url).body[%r{11\.1002/1000/(\d+)}, 1]
+      rescue Mechanize::ResponseCodeError => e
+        raise unless e.response_code == "404" # unknown code => treat as not found
+
+        nil
+      end
+
+      # @return [String] the `rec=` value for the rec.aspx lookup
+      def rec_query
+        ref.suppl ? "#{ref.code} Suppl. #{ref.suppl}" : ref.code
+      end
+
+      # @param idrec [String]
+      # @return [Array<Hash>] editions of the recommendation
+      def editions(idrec)
+        JSON.parse agent.get(format(RECEDITIONS_URL, idrec: idrec)).body
+      rescue JSON::ParserError
+        []
+      end
+
+      # @param edition [Hash] a getRecEditions entry
+      # @return [Relaton::Itu::Hit]
+      def recommendation_hit(edition)
+        Hit.new({
+          code: "ITU-T #{edition['rec_name']}",
+          title: edition["title"],
+          url: format(HANDLE_URL, idrec: edition["idrec"]),
+          type: "recommendation",
+        }, self)
+      end
+
+      # Resolve an ITU-R Radio Regulation or Operational Bulletin to its stable
+      # /pub landing page, whose id is derivable from the reference.
+      def request_publication
+        Util.info "Fetching from www.itu.int ...", key: ref.to_s
+        return @array = [] unless ref.year
+
+        url = "#{DOMAIN}/pub/#{publication_id}"
+        page = fetch_publication_page url
+        return @array = [] if page.nil? || page.uri.to_s.match?(/notfound/i)
+
+        hit = Hit.new({ code: publication_code, title: nil, url: url, type: "publication" }, self)
+        @array = [hit]
+      end
+
+      # @return [Mechanize::Page, nil] nil when the publication does not exist
+      def fetch_publication_page(url)
+        agent.get url
+      rescue Mechanize::ResponseCodeError => e
+        raise unless e.response_code == "404" # unknown publication => not found
+
+        nil
+      end
+
+      # @return [String] the /pub identifier for RR or OB
+      def publication_id
+        if ref.code == "RR"
+          "R-REG-RR-#{ref.year}"
+        else # Operational Bulletin, e.g. OB.1096
+          "T-SP-OB.#{ref.code[/\d+/]}-#{ref.year}"
+        end
+      end
+
+      # @return [String] the docidentifier-friendly code (year only, no month)
+      def publication_code
+        "#{ref.prefix}-#{ref.sector} #{ref.code} (#{ref.year})"
       end
 
       def request_document # rubocop:todo Metrics/MethodLength, Metrics/AbcSize
@@ -50,78 +132,6 @@ module Relaton
         hit.item = item
         @array = [hit]
       end
-
-      # @return [String]
-      def group
-        @group ||= case ref.to_ref
-                   when %r{OB|Operational Bulletin}, %r{^ITU-R\sRR}
-                     "Publications"
-                   when %r{^ITU-T} then "Recommendations"
-                   end
-      end
-
-      # @return [Hash]
-      def params # rubocop:disable Metrics/MethodLength
-        input = ref.dup
-        input.year = nil
-        {
-          "Input" => input.to_s,
-          "Start" => 0,
-          "Rows" => 20,
-          "SortBy" => "RELEVANCE",
-          "ExactPhrase" => false,
-          "CollectionName" => "General",
-          "CollectionGroup" => group,
-          "Sector" => ref.to_ref.match(/(?<=^ITU-)\w/).to_s.downcase,
-          "Criterias" => [{
-            "Name" => "Search in",
-            "Criterias" => [
-              {
-                "Selected" => false,
-                "Value" => "",
-                "Label" => "Name",
-                "Target" => "/name_s",
-                "TypeName" => "CHECKBOX",
-                "GetCriteriaType" => 0,
-              },
-              {
-                "Selected" => false,
-                "Value" => "",
-                "Label" => "Short description",
-                "Target" => "/short_description_s",
-                "TypeName" => "CHECKBOX",
-                "GetCriteriaType" => 0,
-              },
-              {
-                "Selected" => false,
-                "Value" => "",
-                "Label" => "File content",
-                "Target" => "/file",
-                "TypeName" => "CHECKBOX",
-                "GetCriteriaType" => 0,
-              },
-            ],
-            "ShowCheckbox" => true,
-            "Selected" => false,
-          }],
-          "Topics" => "",
-          "ClientData" => {},
-          "Language" => "en",
-          "SearchType" => "All",
-        }
-      end
-
-      # @param data [Hash]
-      # @return [Array<Relaton::Itu::Hit>]
-      def hits(data)
-        data["results"].map do |h|
-          code  = h["Media"]["Name"]
-          title = h["Title"]
-          url   = "#{DOMAIN}#{h['Redirection']}"
-          type  = h["Collection"]["Group"].downcase[0...-1]
-          Hit.new({ code: code, title: title, url: url, type: type }, self)
-        end
-      end
     end
   end
 end

data/lib/relaton/itu/radio_regulations_parser.rb

@@ -19,7 +19,8 @@ module Relaton
       end
 
       def doc_url
-        CGI.unescape(hit.hit[:url]).split("dest=").last
+        url = CGI.unescape(hit.hit[:url])
+        url.include?("dest=") ? url.split("dest=").last : url
       end
 
       def fetch_edition = nil

data/lib/relaton/itu/scraper.rb

@@ -91,10 +91,24 @@ module Relaton
         @docid ||= begin
           docids = hit.hit[:code].to_s.split(" | ").map { |c| createdocid(c) }
           docids << createdocid(doc["rec_name"]) if docids.empty?
+          iso = iso_docid
+          docids << iso if iso
           docids
         end
       end
 
+      # The equivalent ISO/IEC identifier used to come from the search result's
+      # media name; it is now re-sourced from the recommendation header's
+      # `iso_number` field (e.g. "Equivalent standard: ISO/IEC 17788:2014 (Common)").
+      # @return [Relaton::Bib::Docidentifier, nil]
+      def iso_docid
+        return unless idrec # only recommendations carry an idrec / iso_number
+
+        num = parser.doc && parser.doc["iso_number"]
+        id = num && num[%r{ISO(?:/IEC)?(?:/IEEE)?\s+\d[\d-]*}]
+        createdocid(id) if id
+      end
+
       # @param text [String]
       # @return [Relaton::Bib::Docidentifier]
       def createdocid(text) # rubocop:disable Metrics/MethodLength

data/lib/relaton/itu/version.rb

@@ -1,5 +1,5 @@
 module Relaton
   module Itu
-    VERSION = "2.1.1".freeze
+    VERSION = "2.1.2".freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-itu
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 2.1.2
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-05-12 00:00:00.000000000 Z
+date: 2026-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -102,11 +102,6 @@ files:
 - bin/rspec
 - bin/setup
 - data.json
-- grammars/basicdoc.rng
-- grammars/biblio-standoc.rng
-- grammars/biblio.rng
-- grammars/relaton-itu-compile.rng
-- grammars/relaton-itu.rng
 - lib/relaton/itu.rb
 - lib/relaton/itu/bibliography.rb
 - lib/relaton/itu/data_fetcher.rb