relaton 3.0.0.pre.alpha.4 → 3.0.0.pre.alpha.6

data/lib/relaton/oasis/data_parser_utils.rb

@@ -83,7 +83,7 @@ module Relaton
       # @param [#get] agent HTTP client responding to #get(url)
       # @param [Integer] retries number of retries
       #
-      # @return [Nokogiri::HTML::Document, Mechanize::Page, nil] page or nil
+      # @return [Moxml::Document, Mechanize::Page, nil] page or nil
       #
       def retry_page(url, agent, retries = 3)
         sleep 1 # to avoid 429 error
@@ -156,7 +156,7 @@ module Relaton
         href = email[:href]
         if href.start_with?("mailto:")
           [href.split(":")[1]]
-        elsif (cf_email = email.at(".//span[@data-cfemail]"))
+        elsif (cf_email = email.at_xpath(".//span[@data-cfemail]"))
           decoded = decode_cf_email(cf_email["data-cfemail"])
           return [] if decoded.empty?
 

data/lib/relaton/oasis/data_part_parser.rb

@@ -7,7 +7,7 @@ module Relaton
       #
       # Initialize parser.
       #
-      # @param [Nokogiri::HTML::Element] node document node
+      # @param [Moxml::Element] node document node
       #
       def initialize(node, errors = {}, agent: nil)
         @node = node
@@ -20,7 +20,7 @@ module Relaton
 
         sibling_xpath = "./strong/following-sibling::text()" \
                         "|./span[strong]/following-sibling::text()"
-        if @node.at(sibling_xpath)
+        if @node.at_xpath(sibling_xpath)
           nodes_xpath = "./strong/following-sibling::node()" \
                         "|./span[strong]/following-sibling::node()"
           @text = @node.xpath(nodes_xpath).text.strip
@@ -34,7 +34,7 @@ module Relaton
 
         xpath = "./span[@class='citationTitle' " \
                 "or @class='citeTitle']|./em|./i"
-        t = @node.at(xpath)
+        t = @node.at_xpath(xpath)
         @title = if t
                    t.text
                  else
@@ -83,7 +83,7 @@ module Relaton
       # @return [String] document number
       #
       def parse_docnumber
-        ref = @node.at("./span/strong|./strong|./b/span")
+        ref = @node.at_xpath("./span/strong|./strong|./b/span")
         num = ref.text.match(/[^\[\]]+/).to_s
         id = parse_errata(num)
         # some part refs need "Pt" to distinguish from root doc
@@ -183,7 +183,7 @@ module Relaton
       # @return [Array<Bib::Relation>] document relations
       #
       def parse_relation
-        parser = DataParser.new(@node.at("./ancestor::details"), @errors, agent: @agent)
+        parser = DataParser.new(@node.at_xpath("./ancestor::details"), @errors, agent: @agent)
         fref = parser.parse_docid[0].content
         bib = ItemData.new(formattedref: Bib::Formattedref.new(content: fref))
         result = [Bib::Relation.new(type: "partOf", bibitem: bib)]
@@ -221,7 +221,7 @@ module Relaton
       end
 
       def link_node
-        @link_node = @node.at("./a|./following-sibling::p[1]/a")
+        @link_node = @node.at_xpath("./a|./following-sibling::p[1]/a")
       end
 
       #
@@ -230,7 +230,7 @@ module Relaton
       # @return [Array<String>] technology areas
       #
       def parse_technology_area
-        result = super(@node.at("./ancestor::details"))
+        result = super(@node.at_xpath("./ancestor::details"))
         @errors[:part_technology_area] &&= result.empty?
         result
       end

data/lib/relaton/ogc/bibliography.rb

@@ -3,36 +3,37 @@ module Relaton
     module Bibliography
       extend self
 
-      # @param text [String]
+      # @param ref [String, Pubid::Ogc::Identifier]
       # @param year [String, nil]
       # @return [Relaton::Ogc::HitCollection]
-      def search(text, year = nil, _opts = {})
-        code = text.sub(/^OGC\s/, "")
+      def search(ref, year = nil, _opts = {})
+        code = ref.is_a?(String) ? ref.sub(/^OGC\s/, "") : ref
         HitCollection.new(code, year).find
       rescue Faraday::ConnectionFailed, Faraday::SSLError
         raise Relaton::RequestError, HitCollection::ENDPOINT
       end
 
-      # @param code [String]
+      # @param ref [String, Pubid::Ogc::Identifier] a reference, or its parse
+      #   from Relaton::Db (relaton#205), which is used as it is
       # @param year [String, nil]
       # @param opts [Hash]
       # @return [Relaton::Ogc::ItemData, nil]
-      def get(code, year = nil, opts = {})
-        result = bib_search_filter(code, year, opts) || (return nil)
+      def get(ref, year = nil, opts = {})
+        result = bib_search_filter(ref, year, opts) || (return nil)
         ret = bib_results_filter(result, year)
         if ret[:ret]
-          Util.info "Found: `#{ret[:ret].docidentifier.first.content}`", key: code
+          Util.info "Found: `#{ret[:ret].docidentifier.first.content}`", key: ref.to_s
           ret[:ret]
         else
-          fetch_ref_err(code, year, ret[:years])
+          fetch_ref_err(ref, year, ret[:years])
         end
       end
 
       private
 
-      def bib_search_filter(code, year, opts)
-        Util.info "Fetching from Relaton repository ...", key: code
-        search(code, year, opts)
+      def bib_search_filter(ref, year, opts)
+        Util.info "Fetching from Relaton repository ...", key: ref.to_s
+        search(ref, year, opts)
       end
 
       def bib_results_filter(result, year)
@@ -51,11 +52,11 @@ module Relaton
         { years: missed_years }
       end
 
-      def fetch_ref_err(code, year, missed_years)
-        Util.info "Not found.", key: code
+      def fetch_ref_err(ref, year, missed_years)
+        Util.info "Not found.", key: ref.to_s
         unless missed_years.empty?
           Util.info "There was no match for `#{year}`, though there " \
-                    "were matches found for `#{missed_years.join('`, `')}`.", key: code
+                    "were matches found for `#{missed_years.join('`, `')}`.", key: ref.to_s
         end
         nil
       end

data/lib/relaton/ogc/hit_collection.rb

@@ -8,7 +8,7 @@ module Relaton
 
       # @return [self]
       def find
-        return self if ref.nil? || ref.empty?
+        return self if ref.nil? || ref.to_s.empty?
 
         row = best_match ref
         return self unless row
@@ -50,7 +50,7 @@ module Relaton
       # `<nnn>` field alone, so one bucket holds every year that reused the
       # number (`05-007` and `17-007` share bucket `007`).
       #
-      # @param text [String]
+      # @param ref [String, Pubid::Ogc::Identifier]
       # @return [Hash, nil]
       #
       # The substring-scan fallback this used to take when parsing failed is
@@ -59,8 +59,8 @@ module Relaton
       # ids, so an ambiguous reference silently resolved to whichever row
       # sorted first, instead of telling the caller the reference is not an
       # identifier. ecma, w3c and xsf never had one.
-      def best_match(text)
-        pubid = parse_ref text
+      def best_match(ref)
+        pubid = parse_ref ref
         rows = index.search(pubid)
         rows.max_by { |r| [revision_key(r[:id].revision), r[:file]] }
       end
@@ -93,7 +93,7 @@ module Relaton
       # lowercases the revision, so `OGC 19-025r1`, `19-025r1`, `11-038R2` and
       # the revision-less `16-079` all parse without normalization here.
       #
-      # @param text [String]
+      # @param ref [String, Pubid::Ogc::Identifier] a pubid is returned as it is
       # @return [Pubid::Ogc::Identifier, nil]
       #
       # An unrecognized reference **raises**; like ISO, ETSI and 3GPP we let it
@@ -103,8 +103,10 @@ module Relaton
       # logs it through the `StandardError` arm at `lib/relaton/db.rb:122`.
       # Rescuing here would collapse "this identifier is malformed" into "no
       # such document", leaving a caller unable to tell them apart.
-      def parse_ref(text)
-        ::Pubid::Ogc::Identifier.parse text.to_s.strip
+      def parse_ref(ref)
+        return ref unless ref.is_a?(String)
+
+        ::Pubid::Ogc::Identifier.parse ref.strip
       end
     end
   end

data/lib/relaton/ogc/processor.rb

@@ -21,6 +21,19 @@ module Relaton
         Bibliography.get(code, date, opts)
       end
 
+      # OGC documents print their identifier without the `OGC ` token
+      # (`19-025r1`).
+      def bare_identifiers?
+        true
+      end
+
+      # The `year` argument filters by publication year, but an OGC pubid's
+      # `year` is the year in the document number (`19` in `OGC 19-025r1`),
+      # so it cannot be folded into the key: such a query is not cached.
+      def cache_key(ref, year, opts, parsed = nil)
+        year ? nil : super
+      end
+
       # @param source [String]
       # @param opts [Hash]
       def fetch_data(_source, opts)

data/lib/relaton/oiml/processor.rb

@@ -8,6 +8,7 @@ module Relaton
       def initialize
         @short = :relaton_oiml
         @prefix = "OIML"
+        @pubid_identifier = :Oiml # Db cache key
         @defaultprefix = %r{^OIML\s}
         @idtype = "OIML"
       end

data/lib/relaton/omg/bibliography.rb

@@ -6,23 +6,23 @@ module Relaton
     module Bibliography
       extend self
 
-      # @param text [String] the OMG standard reference
+      # @param ref [String, Pubid::Omg::Identifier] the OMG standard reference
       # @return [Relaton::Omg::Item]
-      def search(text)
-        Scraper.scrape_page text
+      def search(ref)
+        Scraper.scrape_page ref
       end
 
-      # @param code [String] the OMG standard reference
+      # @param ref [String, Pubid::Omg::Identifier] the OMG standard reference
       # @param year [String] the year the standard was published (optional)
       # @param opts [Hash] options
       # @return [Relaton::Omg::Item]
-      def get(code, _year = nil, _opts = {})
-        Util.info "Fetching from www.omg.org ...", key: code
-        result = search code
+      def get(ref, _year = nil, _opts = {})
+        Util.info "Fetching from www.omg.org ...", key: ref.to_s
+        result = search ref
         if result
-          Util.info "Found: `#{result.docidentifier.first.content}`", key: code
+          Util.info "Found: `#{result.docidentifier.first.content}`", key: ref.to_s
         else
-          Util.info "Not found.", key: code
+          Util.info "Not found.", key: ref.to_s
         end
         result
       end

data/lib/relaton/omg/scraper.rb

@@ -19,12 +19,13 @@ module Relaton
         @part = part
       end
 
-      # @param ref [String] the OMG reference, e.g. "OMG UML 2.1.1 Superstructure"
+      # @param ref [String, Pubid::Omg::Identifier] the OMG reference, e.g.
+      #   "OMG UML 2.1.1 Superstructure", or its parse from Relaton::Db
       # @return [Relaton::Omg::ItemData, nil] nil when the page is not found
       # @raise [Pubid::Errors::ParseError] when the reference is not an OMG
       #   identifier
       def self.scrape_page(ref)
-        pubid = ::Pubid::Omg::Identifier.parse(ref)
+        pubid = ref.is_a?(String) ? ::Pubid::Omg::Identifier.parse(ref) : ref
         scraper = new(pubid.acronym, pubid.version, pubid.part)
         doc = scraper.get_doc
         return if doc.nil? || scraper.fetch_link.empty?
@@ -63,7 +64,7 @@ module Relaton
       end
 
       def fetch_title
-        content = @doc.at('//dt[.="Title:"]/following-sibling::dd').text
+        content = @doc.at_xpath('//dt[.="Title:"]/following-sibling::dd').text
         content += ": #{@part}" if @part
         [Bib::Title.new(type: "main", content: content, language: "en", script: "Latn")]
       end
@@ -76,7 +77,7 @@ module Relaton
       end
 
       def fetch_abstract
-        content = @doc.at('//section[@id="document-metadata"]/div/div/p').text
+        content = @doc.at_xpath('//section[@id="document-metadata"]/div/div/p').text
         [Bib::Abstract.new(content: content, language: "en", script: "Latn")]
       end
 
@@ -85,7 +86,7 @@ module Relaton
       end
 
       def doc_version
-        @doc_version ||= @doc.at('//dt[.="Version:"]/following-sibling::dd/p/span').text
+        @doc_version ||= @doc.at_xpath('//dt[.="Version:"]/following-sibling::dd/p/span').text
       end
 
       def fetch_date
@@ -105,7 +106,7 @@ module Relaton
       #
       # @return [Date, nil]
       def jsonld_date
-        script = @doc.at('//script[@type="application/ld+json"]')
+        script = @doc.at_xpath('//script[@type="application/ld+json"]')
         return unless script
 
         node = JSON.parse(script.text).find { |e| e.is_a?(Hash) && e[LD_DATE] }
@@ -117,7 +118,7 @@ module Relaton
 
       # @return [Date, nil]
       def dd_date
-        text = @doc.at('//dt[.="Publication Date:"]/following-sibling::dd').text.strip
+        text = @doc.at_xpath('//dt[.="Publication Date:"]/following-sibling::dd').text.strip
         ::Date.parse text
       rescue ::Date::Error
         Util.warn "Cannot parse the publication date `#{text}`."
@@ -125,7 +126,7 @@ module Relaton
       end
 
       def fetch_status
-        status = @doc.at('//dt[.="Document Status:"]/following-sibling::dd')
+        status = @doc.at_xpath('//dt[.="Document Status:"]/following-sibling::dd')
         stage = status.text.strip.match(/\w+/).to_s
         Bib::Status.new(stage: Bib::Status::Stage.new(content: stage))
       end
@@ -135,12 +136,12 @@ module Relaton
 
         @links = []
         if @part
-          a = @doc.at("//a[@href='#{@url}/#{@part}/PDF']")
+          a = @doc.at_xpath("//a[@href='#{@url}/#{@part}/PDF']")
           @links << Bib::Uri.new(type: "src", content: a[:href]) if a
         else
-          a = @doc.at('//dt[.="This Document:"]/following-sibling::dd/a')
+          a = @doc.at_xpath('//dt[.="This Document:"]/following-sibling::dd/a')
           @links << Bib::Uri.new(type: "src", content: a[:href]) if a
-          pdf = @doc.at('//a[@class="download-document"]')
+          pdf = @doc.at_xpath('//a[@class="download-document"]')
           @links << Bib::Uri.new(type: "pdf", content: pdf[:href]) if pdf
         end
         @links
@@ -149,9 +150,9 @@ module Relaton
       def fetch_relation
         v = @doc.xpath('//h2[.="History"]/following-sibling::section/div/table/tbody/tr')
         v.reduce([]) do |mem, row|
-          ver = row.at("td").text
+          ver = row.at_xpath("td").text
           unless ver == doc_version
-            acronym = row.at("td[3]/a")[:href].split("/")[4]
+            acronym = row.at_xpath("td[3]/a")[:href].split("/")[4]
             id = render_id(acronym, ver)
             docid = Docidentifier.new(content: id, type: "OMG")
             bibitem = Bib::ItemBase.new(formattedref: Bib::Formattedref.new(content: id), docidentifier: [docid])

data/lib/relaton/omg.rb

@@ -1,4 +1,4 @@
-require "nokogiri"
+require "moxml"
 require "relaton/bib"
 require_relative "version"
 require_relative "omg/util"

data/lib/relaton/plateau/bibliography.rb

@@ -3,22 +3,24 @@ module Relaton
     module Bibliography
       extend self
 
-      def search(code)
-        HitCollection.new(code).find
+      # @param ref [String, Pubid::Plateau::Identifier] the reference, or the
+      #   parse that Relaton::Db routed with (relaton#205)
+      def search(ref)
+        HitCollection.new(ref).find
       end
 
       # Only a transport failure is rescued, and it becomes the
       # Relaton::RequestError that Relaton::Db retries. Anything else keeps
       # its own class: an unrecognized reference raises Pubid::Errors::ParseError
       # (relaton-cli reports it), and a bug keeps its backtrace.
-      def get(code, _year = nil, _opts = {}) # rubocop:disable Metrics/MethodLength
-        Util.info "Fetching ...", key: code
-        result = search(code).fetch_doc
+      def get(ref, _year = nil, _opts = {}) # rubocop:disable Metrics/MethodLength
+        Util.info "Fetching ...", key: ref.to_s
+        result = search(ref).fetch_doc
         if result
-          Util.info "Found `#{result.docidentifier.first.content}`", key: code
+          Util.info "Found `#{result.docidentifier.first.content}`", key: ref.to_s
           result
         else
-          Util.warn "Not found.", key: code
+          Util.warn "Not found.", key: ref.to_s
         end
       rescue SocketError, Errno::EINVAL, Errno::ECONNRESET, EOFError,
              Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,

data/lib/relaton/plateau/hit_collection.rb

@@ -44,8 +44,9 @@ module Relaton
       # (metanorma/pubid #269) — so search accepts both forms. An unrecognized
       # reference raises; like ISO and 3GPP we let it propagate — relaton-cli
       # rescues Pubid::Errors::Error. See lib/relaton/plateau/CLAUDE.md.
+      # Relaton::Db passes the reference already parsed (relaton#205).
       def pubid_ref
-        @pubid_ref ||= ::Pubid::Plateau.parse(ref)
+        @pubid_ref ||= ref.is_a?(String) ? ::Pubid::Plateau.parse(ref) : ref
       end
 
       # A reference with no edition (`PLATEAU Handbook #00`, or any Technical

data/lib/relaton/plateau/processor.rb

@@ -8,6 +8,7 @@ module Relaton
       def initialize # rubocop:disable Lint/MissingSuper
         @short = :relaton_plateau
         @prefix = "PLATEAU"
+        @pubid_identifier = :Plateau # Db cache key
         @defaultprefix = /^PLATEAU\s/
         @idtype = "PLATEAU"
         @datasets = %w[plateau-handbooks plateau-technical-reports]

data/lib/relaton/un/bibliography.rb

@@ -15,25 +15,36 @@ module Relaton
                 "Could not access #{HitCollection::API_BASE}: #{e.message}"
         end
 
-        # @param ref [String] document reference
+        # @param ref [String, Pubid::Un::Identifier] document reference
         # @param year [String, NilClass]
         # @param opts [Hash] options
         # @return [Relaton::Bib::ItemData]
         def get(ref, _year = nil, _opts = {})
-          Util.info "Fetching from documents.un.org ...", key: ref
-          /^(?:UN\s)?(?<code>.*)/ =~ ref
-          result = isobib_search_filter(code)
-          if result
-            Util.info "Found: `#{result.item.docidentifier[0].content}`", key: ref
-            result.item
-          else
-            Util.info "Not found.", key: ref
-            nil
+          key = ref.to_s
+          Util.info "Fetching from documents.un.org ...", key: key
+          item = isobib_search_filter(document_symbol(ref))&.item
+          unless item
+            Util.info "Not found.", key: key
+            return
           end
+
+          Util.info "Found: `#{item.docidentifier[0].content}`", key: key
+          item
         end
 
         private
 
+        # The UN document symbol to search for. A parsed pubid renders it
+        # (`TRADE/CEFACT/2004/32`); a String may start with a `UN ` token.
+        #
+        # @param ref [String, Pubid::Un::Identifier]
+        # @return [String]
+        def document_symbol(ref)
+          return ref.to_s unless ref.is_a?(String)
+
+          ref[/^(?:UN\s)?(.*)/, 1]
+        end
+
         # Search for hits.
         #
         # @param code [String] reference without correction

data/lib/relaton/un/processor.rb

@@ -10,6 +10,7 @@ module Relaton
       def initialize
         @short = :relaton_un
         @prefix = "UN"
+        @pubid_identifier = :Un # Db cache key
         @defaultprefix = %r{^UN\s}
         @idtype = "UN"
       end

data/lib/relaton/version.rb

@@ -1,3 +1,3 @@
 module Relaton
-  VERSION = "3.0.0.pre.alpha.4".freeze
+  VERSION = "3.0.0.pre.alpha.6".freeze
 end

data/lib/relaton/w3c/processor.rb

@@ -8,6 +8,7 @@ module Relaton
       def initialize # rubocop:disable Lint/MissingSuper
         @short = :relaton_w3c
         @prefix = "W3C"
+        @pubid_identifier = :W3c # Db cache key
         @defaultprefix = %r{^W3C\s}
         @idtype = "W3C"
         @datasets = %w[w3c-api]

data/lib/relaton/xsf/bibliography.rb

@@ -35,16 +35,18 @@ module Relaton
         HitCollection.new(parse_ref(ref)).search
       end
 
-      def get(code, _year = nil, _opts = {})
-        Util.info "Fetching from Relaton repository ...", key: code
-        result = search(code)
+      # @param ref [String, Pubid::Xsf::Identifier] the reference, or the
+      #   parse that Relaton::Db routed with (relaton#205)
+      def get(ref, _year = nil, _opts = {})
+        Util.info "Fetching from Relaton repository ...", key: ref.to_s
+        result = search(ref)
         if result.empty?
-          Util.info "Not found.", key: code
+          Util.info "Not found.", key: ref.to_s
           return
         end
 
         bib = result.first.item
-        Util.info "Found: `#{bib.docidentifier.first.content}`", key: code
+        Util.info "Found: `#{bib.docidentifier.first.content}`", key: ref.to_s
         bib
       end
 
@@ -66,11 +68,14 @@ module Relaton
       # Anything else raises -- see `.search`. `Pubid::Errors::ParseError`
       # includes `Pubid::Errors::Error`, which is the class relaton-cli rescues.
       #
-      # @param ref [String]
+      # @param ref [String, Pubid::Xsf::Identifier]
       # @return [Pubid::Xsf::Identifier]
       # @raise [Pubid::Errors::ParseError]
       #
       def parse_ref(ref)
+        # A parsed pubid comes from Relaton::Db (relaton#205); it is used as it is.
+        return ref unless ref.is_a?(String)
+
         ::Pubid::Xsf::Identifier.parse normalize_ref(ref)
       end
 

data/lib/relaton.rb

@@ -5,6 +5,18 @@
 # `require "relaton/<flavor>"`, loading that flavor's code on demand rather than
 # at `require "relaton"`. When adding a flavor, add an autoload line here.
 require "relaton/version"
+require "moxml"
+
+# YAML through yeptris (Psych-compatible drop-in; stdlib Psych is kept
+# as Yeptris::Psych::ORIGINAL). Must load before any YAML/Psych use.
+# yeptris 0.5 moved the exclusive drop-in rebind to yeptris/psych/drop_in
+# — plain yeptris/psych only defines the namespace and leaves stdlib
+# Psych in place (yeptris-ruby#95, bug 3).
+# Load stdlib Psych first: the drop-in keeps it as Yeptris::Psych::ORIGINAL,
+# and a later plain require "psych" against the rebound constant is a
+# superclass mismatch (yeptris-ruby#95, bug 3 follow-on).
+require "psych"
+require "yeptris/psych/drop_in"
 
 module Relaton
   autoload :Logger, "relaton/logger"
@@ -46,6 +58,7 @@ module Relaton
   autoload :Plateau, "relaton/plateau"
   autoload :Doi, "relaton/doi"
   autoload :Sdo, "relaton/sdo"
+  autoload :Cloud, "relaton/cloud"
 
   # Global PubID prefix register (relaton-db#103). Maps an SDO document-ID
   # prefix to the flavor module(s) that own it. Conflicting prefixes (e.g.