relaton 3.0.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/itu/recommendation_parser.rb

@@ -1,161 +1,30 @@
+require_relative "recommendation_fields"
+
 module Relaton
   module Itu
+    # Live-lookup ITU-T Recommendation parser. All field extraction lives in the
+    # shared RecommendationFields module (reused by the offline DataParserT
+    # harvester); this class only binds the module's `agent`/`idrec`/`imp` hooks.
     class RecommendationParser
-      include Relaton::Core::ArrayWrapper
-
-      RECHDR = "https://www.itu.int/mws/api/recommendations/getRecHdrDetail?idrec=%{idrec}&lang=en"
-      RECEDITIONS = "https://www.itu.int/mws/api/recommendations/getRecEditions?idrec=%{idrec}&lang=en"
-      RECSUPPLEMENTS = "https://www.itu.int/mws/api/recommendations/getRecSupplements?idrec=%{idrec}&lang=en"
-      IMPLGUIDES = "https://www.itu.int/mws/api/recommendations/getImplGuides?idrec=%{idrec}&lang=en"
-
-      def initialize(hit, idrec, imp)
-        @hit = hit
+      include RecommendationFields
+
+      # @param agent [Mechanize] the HTTP agent (was a Hit before decoupling)
+      # @param idrec [String, Integer] the recommendation record id
+      # @param imp [Boolean] Implementers' Guide flag
+      # @param cache [#fetch, #warm] cross-row cache for the family-invariant
+      #   endpoints. Defaults to NullCache, i.e. off — only the offline
+      #   harvester passes a real one, so the live lookup path keeps exactly the
+      #   behaviour and the request count it had before the cache existed.
+      def initialize(agent, idrec, imp, cache: NullCache.instance)
+        @agent = agent
         @idrec = idrec
         @imp = imp
-      end
-
-      def doc
-        @doc ||= begin
-          url = (imp ? IMPLGUIDES : RECHDR) % { idrec: idrec }
-          resp = get_data url
-          imp ? resp.first : resp
-        end
-      end
-
-      # @return [String, nil]
-      def fetch_edition
-        self_edition&.dig("Version")
-      end
-
-      # @return [Array<Relaton::Bib::Title>]
-      def fetch_titles
-        title = imp ? doc["imp_title_e"] : doc["rec_title"]
-        return [] if title.nil? || title.empty?
-
-        Relaton::Bib::Title.from_string title, "en", "Latn"
-      end
-
-      # @return [Relaton::Bib::Status, nil]
-      def fetch_status
-        inforce = imp ? imp_status : doc["status"]
-        return if inforce.nil? || inforce.empty?
-
-        status = inforce == "In force" ? "Published" : "Withdrawal"
-        Relaton::Bib::Status.new(stage: Relaton::Bib::Status::Stage.new(content: status))
-      end
-
-      # @return [Array<Relaton::Bib::Date>]
-      def fetch_dates
-        array(doc_date).map { |on| Relaton::Bib::Date.new(type: "published", at: on) }
-      end
-
-      # @return [Array<Relaton::Bib::LocalizedMarkedUpString>]
-      def fetch_abstract
-        array(doc["summary"]).map do |content|
-          Relaton::Bib::Abstract.new(content: content, language: "en", script: "Latn")
-        end
-      end
-
-      # @return [Array<Relaton::Bib::Uri>]
-      def fetch_source
-        link = imp ? doc["imp_dms_link"] : doc["handle_id"]
-        links = [Relaton::Bib::Uri.new(type: "src", content: link)]
-        links << Relaton::Bib::Uri.new(type: "pdf", content: doc["handle_id_pdf_link"]) if doc["handle_id_pdf_link"]
-        imp_word_link { |wlink| links << Relaton::Bib::Uri.new(type: "word", content: wlink) }
-        links
-      end
-
-      def doc_date
-        return @doc_date if defined? @doc_date
-
-        date = imp ? doc["imp_approval_date"] : doc["approval_date"]
-        @doc_date = Date.parse(date).to_s rescue date # rubocop:disable Style/RescueModifier
-      end
-
-      # @return [Array<Relaton::Bib::Relation>]
-      def fetch_relations
-        relations = []
-        editions.each do |ed|
-          next if ed["idrec"] == idrec
-
-          relations << create_relation("hasEdition", ed["title"], ed["rec_name"])
-        end
-
-        supplements.each { |supp| relations << create_relation("complementOf", supp["title_text"], supp["rec_name"]) }
-        relations
-      end
-
-        # Fetch the study group name from the recommendation HTML page.
-      # @return [String, nil]
-      def fetch_workgroup
-        url = "https://www.itu.int/ITU-T/recommendations/rec.aspx?rec=#{idrec}&lang=en"
-        page = hit.hit_collection.agent.get(url)
-        wg = page.at('//span[contains(@id, "uc_rec_main_info1_rpt_main_ctl00_Label8")]/a')
-        wg&.text
-      rescue StandardError
-        nil
+        @cache = cache
       end
 
       private
 
-      attr_reader :hit, :idrec, :imp
-
-      def get_data(url)
-        JSON.parse request_document(url).body
-      end
-
-      def request_document(url)
-        hit.hit_collection.agent.get url
-      rescue Mechanize::ResponseCodeError, SocketError, Timeout::Error, Errno::ECONNRESET,
-              EOFError, Net::ProtocolError, OpenSSL::SSL::SSLError => e
-        raise Relaton::RequestError, "Could not access #{url}: #{e.message}"
-      end
-
-      def editions
-        @editions ||= begin
-          url = RECEDITIONS % { idrec: idrec }
-          get_data(url) || []
-        end
-      end
-
-      def self_edition
-        @self_edition ||= editions.find { |ed| ed["idrec"] == idrec }
-      end
-
-      def imp_status
-        self_edition&.dig("status")
-      end
-
-      def imp_word_link
-        return unless doc["imp_dms_link"]
-
-        @doc_page ||= request_document(doc["imp_dms_link"])
-        wrd_elm = @doc_page.at("//font[contains(.,'Word')]/../..")
-        yield wrd_elm[:href] if block_given? && wrd_elm
-      end
-
-      def create_relation(type, title_text, id)
-        titles = []
-        if title_text && !title_text.empty?
-          titles << Relaton::Bib::Title.new(content: title_text, language: "en", script: "Latn")
-        end
-
-        fref = titles.empty? ? id : nil
-        did = Relaton::Bib::Docidentifier.new(type: "ITU", content: id, primary: true)
-        bibitem = Relaton::Bib::ItemData.new(title: titles, formattedref: (fref ? Relaton::Bib::Formattedref.new(content: fref) : nil), docidentifier: [did])
-        Relaton::Bib::Relation.new(type: type, bibitem: bibitem)
-      end
-
-      def supplements
-        @supplements ||= begin
-          if imp
-            []
-          else
-            url = RECSUPPLEMENTS % { idrec: idrec }
-            get_data(url) || []
-          end
-        end
-      end
+      attr_reader :agent, :idrec, :imp, :cache
     end
   end
 end

data/lib/relaton/itu/scraper.rb

@@ -78,19 +78,29 @@ module Relaton
       def parser
         @parser ||= begin
           if idrec
-            RecommendationParser.new hit, idrec, imp
+            RecommendationParser.new hit.hit_collection.agent, idrec, imp
           else
             RadioRegulationsParser.new hit
           end
         end
       end
 
-      # Fetch docid.
+      # The primary docid comes from the hit's code; a recommendation additionally
+      # carries the equivalent ISO/IEC(/IEEE) identifier, which used to be part of
+      # the search result's media name and now comes from the recommendation
+      # header's `iso_number` via the shared `RecommendationFields#iso_docid`.
+      # Both are guarded on `idrec` — `RadioRegulationsParser` (RR/OB) parses an
+      # HTML page and has neither `rec_name` nor `iso_docid`.
+      #
       # @return [Array<Relaton::Bib::Docidentifier>]
       def docid
         @docid ||= begin
           docids = hit.hit[:code].to_s.split(" | ").map { |c| createdocid(c) }
-          docids << createdocid(doc["rec_name"]) if docids.empty?
+          if idrec
+            docids << createdocid("ITU-T #{parser.doc['rec_name']}") if docids.empty?
+            iso = parser.iso_docid
+            docids << iso if iso
+          end
           docids
         end
       end

data/lib/relaton/itu.rb

@@ -1,5 +1,6 @@
 require "mechanize"
 require "parslet"
+require "pubid"
 require "digest/md5"
 require "relaton/index"
 require "relaton/bib"
@@ -17,7 +18,7 @@ require_relative "itu/bibliography"
 
 module Relaton
   module Itu
-    INDEXFILE = "index-v1"
+    INDEXFILE = "index-v2"
 
     class Error < StandardError; end
 

data/lib/relaton/jcgm/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Jcgm
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/jcgm/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Jcgm
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/jcgm/bibliography.rb

@@ -0,0 +1,97 @@
+require "net/http"
+
+module Relaton
+  module Jcgm
+    # Retrieval for JCGM (Joint Committee for Guides in Metrology) publications.
+    # Index-backed (no scraping): the reference is parsed with `Pubid::Jcgm`,
+    # matched against the pre-built index (whose rows are `Pubid::Jcgm::Identifier`
+    # hashes, deserialized via `pubid_class`), and the matching per-document YAML
+    # is fetched from the `relaton/relaton-data-jcgm` GitHub repo.
+    module Bibliography
+      ENDPOINT = "https://raw.githubusercontent.com/relaton/relaton-data-jcgm/main/".freeze
+
+      class << self
+        #
+        # Search for a JCGM publication by its identifier.
+        #
+        # @param text [String, Pubid::Jcgm::Identifier] e.g. "JCGM 200:2012" or
+        #   "JCGM 17th Meeting (2012)"
+        # @param year [String, nil] edition year (optional; may be embedded)
+        # @param _opts [Hash] options (unused)
+        #
+        # @return [Relaton::Jcgm::Item, nil]
+        #
+        def search(text, year = nil, _opts = {}) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+          pubid = parse_ref(text)
+          Util.info "Fetching from Relaton repository ...", key: pubid.to_s
+          row = index.search(pubid) { |r| pubid_match?(r[:id], pubid, year) }
+                     .max_by { |r| r[:id].year.to_i }
+          unless row
+            Util.info "Not found.", key: pubid.to_s
+            return
+          end
+
+          uri = URI("#{ENDPOINT}#{row[:file]}")
+          resp = Net::HTTP.get_response uri
+          unless resp.code == "200"
+            raise Relaton::RequestError, "Could not access #{uri}: HTTP #{resp.code}"
+          end
+
+          item = Relaton::Jcgm::Item.from_yaml resp.body
+          Util.info "Found: `#{item.docidentifier.first&.content}`", key: pubid.to_s
+          item.tap { |i| i.fetched = Date.today.to_s }
+        rescue SocketError, Errno::EINVAL, Errno::ECONNRESET, EOFError,
+               Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
+               Net::ProtocolError, Net::ReadTimeout, OpenSSL::SSL::SSLError,
+               Errno::ETIMEDOUT => e
+          raise Relaton::RequestError, "Could not access #{uri}: #{e.message}"
+        end
+
+        # @see #search
+        def get(ref, year = nil, opts = {})
+          search(ref, year, opts)
+        end
+
+        private
+
+        # Parse a reference into a Pubid::Jcgm identifier. An unrecognized
+        # reference raises; like ISO and 3GPP we let it propagate --
+        # relaton-cli rescues Pubid::Errors::Error and renders "... is not a
+        # recognized standards identifier". The `search` rescue lists transport
+        # errors only, so it does not swallow the parse error.
+        def parse_ref(text)
+          return text unless text.is_a?(String)
+
+          ::Pubid::Jcgm.parse(text)
+        end
+
+        def index
+          Relaton::Index.find_or_create(
+            :jcgm,
+            url: "#{ENDPOINT}#{INDEXFILE}.zip",
+            file: "#{INDEXFILE}.yaml",
+            pubid_class: ::Pubid::Jcgm::Identifier,
+          )
+        end
+
+        # `row_id` and `query` are both `Pubid::Jcgm::Identifier` instances.
+        # Matching is pubid's asymmetric subset match (`Pubid::SubsetMatch`):
+        # the reference on the left, the row on the right, and a component the
+        # reference omits matches any value. Guides distinguish editions by year
+        # (`JCGM 200:2008` vs `:2012`), so a reference without a year reaches
+        # both and the latest edition is picked by `max_by` in #search. Meetings
+        # distinguish by number (`17th` vs `18th`), which the reference states,
+        # so they never collapse together. The type is the identifier's class,
+        # and `===` requires the same class, so cross-type collisions cannot
+        # occur. The `year` argument lets a caller pin an edition the reference
+        # string omitted; a year the reference states is compared by `===`
+        # itself.
+        def pubid_match?(row_id, query, year)
+          wanted_year = (query.year || year)&.to_s
+          query === row_id &&
+            (wanted_year.nil? || row_id.year.to_s == wanted_year)
+        end
+      end
+    end
+  end
+end

data/lib/relaton/jcgm/data_fetcher.rb

@@ -0,0 +1,81 @@
+require_relative "../jcgm"
+require_relative "meetings_parser"
+
+module Relaton::Jcgm
+  # Harvests the JCGM **meeting proceedings** from the `jcgm/meetings-{en,fr}`
+  # dirs of a local checkout of metanorma/bipm-data-outcomes, writing one Relaton
+  # YAML per meeting and indexing each by its `Pubid::Jcgm` identifier.
+  #
+  # The curated guide/GUM/VIM records are **not** harvested here: they are
+  # hand-maintained YAMLs that live in the data repo's own `static/` dir, so the
+  # data repo's `crawler.rb` indexes them (like relaton-data-bipm's crawler does)
+  # by looping `static/**` and calling the public, guarded {#add_to_index} —
+  # keeping the "which static files" policy in the data repo while the guarded,
+  # sorted, `_type: pubid:jcgm:*` indexing mechanism stays here.
+  class DataFetcher < Relaton::Core::DataFetcher
+    attr_reader :output, :format, :ext, :files, :errors
+
+    def index
+      @index ||= Relaton::Index.find_or_create(
+        :jcgm, file: "#{INDEXFILE}.yaml", pubid_class: ::Pubid::Jcgm::Identifier
+      )
+    end
+
+    def log_error(msg) = Util.error msg
+
+    #
+    # @param [String] source dataset name ("bipm-data-outcomes")
+    #
+    def fetch(source)
+      MeetingsParser.parse(self) if source == "bipm-data-outcomes"
+      index.save
+      report_errors
+    end
+
+    #
+    # Save document to file
+    #
+    # @param [String] path Path to file
+    # @param [Relaton::Jcgm::ItemData] item document to save
+    # @param [Boolean, nil] warn_duplicate Warn if document already exists
+    #
+    def write_file(path, item, warn_duplicate: true)
+      content = serialize item
+      if @files.include?(path)
+        Util.warn "File #{path} already exists" if warn_duplicate
+      else
+        @files << path
+      end
+      File.write path, content, encoding: "UTF-8"
+    end
+
+    #
+    # Index a document by its pubid. Guarded defensively: a docnumber the
+    # `Pubid::Jcgm` grammar can't parse is skipped with a warning rather than
+    # allowed to corrupt the whole index (`Relaton::Index` rejects an index if a
+    # single row fails to deserialize). All current JCGM records parse on pubid
+    # main; the guard protects against a future malformed record.
+    #
+    # Public so the data repo's crawler can index its curated `static/` guides
+    # through the same guarded, sorted, pubid-backed path the meetings use.
+    #
+    # @param [String] docnumber the record's docnumber (e.g. "JCGM 17th Meeting (2012)")
+    # @param [String] path repo-relative path to the YAML file
+    #
+    def add_to_index(docnumber, path)
+      # Store the Pubid::Jcgm identifier object (not its hash): with pubid_class
+      # set, Relaton::Index then sorts the index by id number on save and
+      # serializes each id to its `_type: pubid:jcgm:*` hash, so the published
+      # index is already sorted (no "not sorted by id number" warning on fetch).
+      index.add_or_update ::Pubid::Jcgm.parse(docnumber), path
+    rescue StandardError => e
+      Util.warn "Skipping index entry for `#{docnumber}` (#{path}): #{e.message}"
+    end
+
+    def to_xml(item) = item.to_xml bibdata: true
+
+    def to_yaml(item) = item.to_yaml
+
+    def to_bibxml(item) = item.to_rfcxml
+  end
+end

data/lib/relaton/jcgm/docidentifier.rb

@@ -0,0 +1,102 @@
+module Relaton
+  module Jcgm
+    # A JCGM document identifier, backed by `Pubid::Jcgm` (mirrors OIML's
+    # pubid-backed Docidentifier). Parsing is best-effort: forms pubid doesn't
+    # recognise (e.g. the combined bilingual docidentifiers and the French
+    # `réunion` form) leave `pubid` nil via the `content=` rescue rather than
+    # raising during deserialization.
+    class Docidentifier < Bib::Docidentifier
+      attr_reader :pubid
+
+      # Capture the inherited content setter before overriding #content=, so
+      # #refresh_content! can write the re-rendered string back WITHOUT
+      # re-parsing (a re-parse would rebuild @pubid from the string, which may
+      # not even parse back — e.g. an all-parts wrapper's own "(all parts)"
+      # rendering — and would discard whatever mutation was just made).
+      alias_method :store_content, :content=
+
+      def initialize(attrs = {}, options = {})
+        pubid = attrs.is_a?(Hash) ? attrs.delete(:pubid) : nil
+        attrs[:content] ||= pubid.to_s if pubid
+        super
+        @pubid = pubid if pubid
+      end
+
+      def content=(value)
+        super
+        @pubid = ::Pubid::Jcgm.parse(value) if value
+      rescue StandardError
+        @pubid = nil
+      end
+
+      # The class carried `#pubid` but none of `Bib::Docidentifier`'s abstract
+      # `remove_part!` / `remove_date!` / `to_all_parts!`, so it looked migrated
+      # while still raising `NotImplementedError` on every JCGM item through
+      # `Bib::ItemData#to_all_parts` / `#to_most_recent_reference` — and
+      # `NotImplementedError` descends from `ScriptError`, so a caller's
+      # `rescue => e` did not catch it.
+      #
+      # `Pubid::Jcgm` keeps the publication date in `date` (a
+      # `Components::Date`, unlike EASC's and GOST's plain `year`), so
+      # `remove_date!` clears that: `JCGM 100:2008` -> `JCGM 100`. `part` and
+      # `subpart` come from the pubid base class and are unused by JCGM today,
+      # so `remove_part!` is a no-op for the rendered string — implemented so it
+      # never raises.
+      #
+      # The mutation is applied to the identifier **and to its `#root`**. A
+      # supplement/corrigendum wraps the document it amends in `base`, and it
+      # exposes its own `date=` while carrying nil there — the year lives on the
+      # base. Clearing only the wrapper is therefore a silent no-op:
+      # `JCGM 200:2008 Corrigendum` keeps its 2008. Clearing the root as well
+      # gives `JCGM 200 Corrigendum`. `#root` is the same accessor
+      # `Relaton::Index::Type#candidates_by_number` narrows on, so the two stay
+      # consistent.
+      #
+      # Every write is guarded by `respond_to?`, and all three no-op when
+      # `@pubid` is nil — the common case the `content=` rescue exists for (the
+      # combined bilingual docidentifiers and the French `réunion` form).
+
+      def remove_part!
+        clear_attr!(:part)
+      end
+
+      def remove_date!
+        clear_attr!(:date)
+      end
+
+      def to_all_parts!
+        return if !@pubid || @pubid.all_parts?
+
+        # `#exclude` (no args) rebuilds a full independent copy, including
+        # the `#root` of a supplement/corrigendum — `remove_part!`/
+        # `remove_date!` mutate BOTH `@pubid` and its root in place (see
+        # `clear_attr!`), so even a shallow `.dup` would still lose the
+        # original date to that mutation.
+        original = @pubid.exclude
+        remove_part!
+        remove_date!
+        @pubid = original.to_all_parts
+      end
+
+      private
+
+      def clear_attr!(attr)
+        return unless @pubid
+
+        targets = [@pubid]
+        targets << @pubid.root if @pubid.respond_to?(:root) && !@pubid.root.equal?(@pubid)
+        targets.each do |id|
+          next unless id.respond_to?("#{attr}=")
+
+          id.public_send("#{attr}=", nil)
+          id.subpart = nil if attr == :part && id.respond_to?(:subpart=)
+        end
+        refresh_content!
+      end
+
+      def refresh_content!
+        store_content(@pubid.to_s) if @pubid
+      end
+    end
+  end
+end

data/lib/relaton/jcgm/doctype.rb

@@ -0,0 +1,12 @@
+module Relaton
+  module Jcgm
+    # JCGM document types. Meetings are `meeting-report`; the guides/GUM/VIM
+    # records carry `international-standard` (the shape they were filed with
+    # under BIPM). The list is intentionally not enforced as an enum so any
+    # unforeseen dataset value still round-trips (mirrors OIML's Doctype).
+    class Doctype < Bib::Doctype
+      TYPES = %w[meeting-report guide international-standard vocabulary
+                 monographie rapport].freeze
+    end
+  end
+end

data/lib/relaton/jcgm/ext.rb

@@ -0,0 +1,23 @@
+require_relative "doctype"
+require_relative "structured_identifier"
+
+module Relaton
+  module Jcgm
+    # JCGM bibliographic item extension. JCGM records are BIPM-shaped, so this
+    # mirrors `Relaton::Bipm::Ext`: a JCGM `Doctype` and a single
+    # `structuredidentifier` (overriding the base collection). `flavor`,
+    # `schema_version`, `subdoctype`, `ics` come from the shared `Bib::Ext`.
+    class Ext < Bib::Ext
+      attribute :doctype, Doctype
+      attribute :structuredidentifier, StructuredIdentifier
+
+      xml do
+        map_element "structuredidentifier", to: :structuredidentifier
+      end
+
+      key_value do
+        map_element "structuredidentifier", to: :structuredidentifier
+      end
+    end
+  end
+end

data/lib/relaton/jcgm/item.rb

@@ -0,0 +1,20 @@
+require_relative "item_data"
+require_relative "ext"
+require_relative "docidentifier"
+
+module Relaton
+  module Jcgm
+    class Item < Bib::Item
+      model ItemData
+
+      attribute :ext, Ext
+      attribute :docidentifier, Docidentifier, collection: true,
+                                               initialize_empty: true
+    end
+  end
+end
+
+require_relative "relation"
+
+Relaton::Jcgm::Item.attribute :relation, Relaton::Jcgm::Relation,
+                              collection: true, initialize_empty: true

data/lib/relaton/jcgm/item_base.rb

@@ -0,0 +1,18 @@
+module Relaton
+  module Jcgm
+    class ItemBase < Lutaml::Model::Serializable
+      include Bib::NamespaceHelper
+
+      attr_accessor :type
+
+      model ItemData
+
+      instance_exec(&Bib::ItemShared::ATTRIBUTES)
+
+      xml do
+        map_attribute "type", to: :type
+        instance_exec(&Bib::ItemShared::XML_BODY)
+      end
+    end
+  end
+end

data/lib/relaton/jcgm/item_data.rb

@@ -0,0 +1,6 @@
+module Relaton
+  module Jcgm
+    class ItemData < Bib::ItemData
+    end
+  end
+end