relaton 3.0.0.pre.alpha.4 → 3.0.0.pre.alpha.6

data/lib/relaton/cie/bibliography.rb

@@ -5,23 +5,25 @@ module Relaton
   # IETF bibliography module
     module Bibliography
       class << self
-        # @param code [String] the ECMA standard Code to look up (e..g "ECMA-6")
+        # @param ref [String, Pubid::Cie::Identifier] the CIE reference (e.g.
+        #   "CIE 001-1980"), or its parse from Relaton::Db (relaton#205)
         # @return [Relaton::Cie::ItemData]
-        def search(code)
-          Scrapper.scrape_page code
+        def search(ref)
+          Scrapper.scrape_page ref
         end
 
-        # @param code [String] the ECMA standard Code to look up (e..g "ECMA-6")
+        # @param ref [String, Pubid::Cie::Identifier] the CIE reference (e.g.
+        #   "CIE 001-1980"), or its parse from Relaton::Db (relaton#205)
         # @param year [String] not used
         # @param opts [Hash] not used
         # @return [Relaton::Cie::ItemData] Relaton of reference
-        def get(code, _year = nil, _opts = {})
-          Util.info "Fetching from Relaton repository ...", key: code
-          result = search code
+        def get(ref, _year = nil, _opts = {})
+          Util.info "Fetching from Relaton repository ...", key: ref.to_s
+          result = search ref
           if result
-            Util.info "Found: `#{result.docidentifier.first.content}`", key: code
+            Util.info "Found: `#{result.docidentifier.first.content}`", key: ref.to_s
           else
-            Util.info "Not found.", key: code
+            Util.info "Not found.", key: ref.to_s
           end
           result
         end

data/lib/relaton/cie/data_fetcher.rb

@@ -3,7 +3,7 @@
 require "English"
 require "fileutils"
 require "ferrum"
-require "nokogiri"
+require "moxml"
 require "pubid"
 require "relaton/index"
 require "relaton/bib"
@@ -58,7 +58,7 @@ module Relaton
       def get(url)
         @browser.go_to(url)
         wait_for_challenge
-        Nokogiri::HTML(@browser.body)
+        Moxml.new.parse_html(@browser.body)
       end
 
       def quit
@@ -181,7 +181,7 @@ module Relaton
         Util.error msg
       end
 
-      # @param hit [Nokogiri::HTML::Document]
+      # @param hit [Moxml::Document]
       # @param doc [Mechanize::Page]
       # @return [Array<Relaton::Bib::Docidentifier>]
       def fetch_docid(hit, doc)
@@ -200,7 +200,7 @@ module Relaton
         else
           @errors[:docid_2] &&= true
         end
-        isbn = doc.at('//h3[contains(.,"ISBN")]/following-sibling::span')&.text
+        isbn = doc.at_xpath('//h3[contains(.,"ISBN")]/following-sibling::span')&.text
         if isbn && !isbn.strip.empty?
           docid << Bib::Docidentifier.new(type: "ISBN", content: isbn)
           @errors[:docid_isbn] &&= false
@@ -211,7 +211,7 @@ module Relaton
       end
 
       def parse_code(hit, doc = nil)
-        code = hit.at("h3/a").text.strip.squeeze(" ").sub(/\u25b9/, "").gsub(" / ", "/")
+        code = hit.at_xpath(".//h3/a").text.strip.squeeze(" ").sub(/\u25b9/, "").gsub(" / ", "/")
         c2idx = %r{(?:\(|/)(?<c2>(?:ISO|IEC)\s[^()]+)} =~ code
         code = code[0...c2idx].strip if c2idx
         [primary_code(code, doc), c2]
@@ -221,7 +221,7 @@ module Relaton
         /^(?<code1>[^(]+)(?:\((?<code2>[a-zA-Z]+\d+,(?:\sPages)?[^)]+))?/ =~ code
         if code1&.match?(/^CIE/)
           parse_cie_code code1, code2, doc
-        elsif (pcode = doc&.at('//h3[.="Product Code(s):"]/following-sibling::span'))
+        elsif (pcode = doc&.at_xpath('//h3[.="Product Code(s):"]/following-sibling::span'))
           "CIE #{pcode.text.strip.match(/[^,]+/)}"
         else
           num = code.match(/(?<=\()\w{2}\d+,.+(?=\))/).to_s.gsub(/,(?=\s)/, "").gsub(/,(?=\S)/, " ")
@@ -231,7 +231,7 @@ module Relaton
 
       def parse_cie_code(code1, code2, doc = nil) # rubocop:disable Metrics/CyclomaticComplexity
         code = code1.size > 25 && code2 ? "CIE #{code2.sub(/,(\sPages)?/, '')}" : code1
-        add = doc&.at("//hgroup/h2")&.text&.match(/(Add)endum\s(\d+)$/)
+        add = doc&.at_xpath("//hgroup/h2")&.text&.match(/(Add)endum\s(\d+)$/)
         return code unless add
 
         "#{code} #{add[1]} #{add[2]}"
@@ -244,7 +244,7 @@ module Relaton
       # @param doc [Mechanize::Page]
       # @return [Array<Relaton::Bib::Title>]
       def fetch_title(doc)
-        t = doc.at("//hgroup/h2/text()", "//hgroup/h1/text()")
+        t = doc.at_xpath("//hgroup/h2/text()|//hgroup/h1/text()")
         unless t && !t.text.strip.empty?
           @errors[:title] &&= true
           return []
@@ -270,7 +270,7 @@ module Relaton
       # @param doc [Mechanize::Page]
       # @return [String]
       def fetch_edition(doc)
-        ed = doc.at("//h3[.='Edition:']/following-sibling::span")
+        ed = doc.at_xpath("//h3[.='Edition:']/following-sibling::span")
         @errors[:edition] &&= true
         return unless ed
 
@@ -285,16 +285,16 @@ module Relaton
       # @return [Array<Relaton::Cie::Relation>]
       def fetch_relation(doc) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
         rels = doc.xpath('//section[@class="history"]/ol/li[not(contains(@class,"selected-product"))]').map do |rel|
-          ref = rel.at("a")
+          ref = rel.at_xpath(".//a")
           url = "https://www.techstreet.com#{ref[:href]}"
-          title = Bib::Title.from_string ref.at('p/span[@class="title"]').text
-          did = ref.at("h3").text
+          title = Bib::Title.from_string ref.at_xpath('.//p/span[@class="title"]').text
+          did = ref.at_xpath(".//h3").text
           docid = [Bib::Docidentifier.new(type: "CIE", content: did, primary: true)]
-          on = ref.at("p/time")
+          on = ref.at_xpath(".//p/time")
           date = [Bib::Date.new(type: "published", at: on[:datetime])]
           source = [Bib::Uri.new(type: "src", content: url)]
           bibitem = ItemData.new docidentifier: docid, title: title, source: source, date: date
-          type = ref.at('//li/i[contains(@class,"historical")]') ? "updates" : "updatedBy"
+          type = ref.at_xpath('//li/i[contains(@class,"historical")]') ? "updates" : "updatedBy"
           Bib::Relation.new(type: type, bibitem: bibitem)
         end
         @errors[:relation] &&= rels.empty?
@@ -313,7 +313,7 @@ module Relaton
       # @param doc [Mechanize::Page]
       # @return [Array<Relaton::Bib::LocalizedMarkedUpString>]
       def fetch_abstract(doc)
-        content = doc.at('//div[contains(@class,"description")]')&.text&.strip
+        content = doc.at_xpath('//div[contains(@class,"description")]')&.text&.strip
         if content.nil? || content.empty?
           @errors[:abstract] &&= true
           return []
@@ -422,10 +422,10 @@ module Relaton
       # workers. Called directly (single stubbed agent) by specs and from the
       # pool with a per-worker agent/pacing/position.
       #
-      # @param hit [Nokogiri::HTML::Element]
+      # @param hit [Moxml::Element]
       # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       def parse_page(hit, pos = nil, worker_agent = agent, pacing = nil)
-        url = hit.at('h3/a')[:href]
+        url = hit.at_xpath('.//h3/a')[:href]
         doc = time_req(pacing) { worker_agent.get url }
         @mutex.synchronize do
           item = ItemData.new(
@@ -460,7 +460,7 @@ module Relaton
       def collect_hits(url = URL, hits = [])
         result = time_req(listing_pacing) { agent.get url }
         hits.concat result.xpath("//li[@data-product]").to_a
-        np = result.at '//a[@class="next_page"]'
+        np = result.at_xpath '//a[@class="next_page"]'
         return hits unless np
 
         next_href = np[:href]

data/lib/relaton/cie/processor.rb

@@ -6,6 +6,7 @@ module Relaton
       def initialize # rubocop:disable Lint/MissingSuper
         @short = :relaton_cie
         @prefix = "CIE"
+        @pubid_identifier = :Cie # Db cache key
         @defaultprefix = /^CIE(-|\s)/
         @idtype = "CIE"
         @datasets = %w[cie-techstreet]

data/lib/relaton/cie/scrapper.rb

@@ -6,14 +6,15 @@ module Relaton
       ENDPOINT = "https://raw.githubusercontent.com/relaton/relaton-data-cie/refs/heads/v2/".freeze
 
       class << self
-        # @param code [String]
+        # @param ref [String, Pubid::Cie::Identifier] the reference, or the
+        #   parse that Relaton::Db routed with (relaton#205)
         # @return [Relaton::Cie::ItemData]
-        def scrape_page(code)
+        def scrape_page(ref)
           # An unrecognized reference raises; like ISO and 3GPP we let it
           # propagate -- relaton-cli rescues Pubid::Errors::Error and renders
           # "... is not a recognized standards identifier". Partial refs
           # (`CIE 001`, `CIE 15`) parse, so nothing valid is lost.
-          pubid = ::Pubid::Cie.parse code
+          pubid = to_pubid ref
           index = Index.find_or_create :cie, url: "#{ENDPOINT}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml",
                                               pubid_class: ::Pubid::Cie::Identifier
           # Pass the parsed pubid (not the raw String) so index-v2 narrows
@@ -26,25 +27,31 @@ module Relaton
                      .min_by { |r| r[:id].to_s }
           return unless row
 
-          parse_page "#{ENDPOINT}#{row[:file]}", code
+          parse_page "#{ENDPOINT}#{row[:file]}", ref.to_s
         end
 
         private
 
+        # @param ref [String, Pubid::Cie::Identifier]
+        # @return [Pubid::Cie::Identifier] parses only a String
+        def to_pubid(ref)
+          ref.is_a?(String) ? ::Pubid::Cie.parse(ref) : ref
+        end
+
         # @param url [String]
-        # @param code [String]
+        # @param ref [String]
         # @retrurn [Relato::Cie::ItemData]
-        def parse_page(url, code)
+        def parse_page(url, ref)
           resp = Mechanize.new.get url
           Item.from_yaml(resp.body).tap { |item| item.fetched = Date.today.to_s }
         rescue Mechanize::ResponseCodeError => e
           return if e.response_code == "404"
 
-          raise Relaton::RequestError, "No document found for #{code} reference. #{e.message}"
+          raise Relaton::RequestError, "No document found for #{ref} reference. #{e.message}"
         rescue Mechanize::RedirectLimitReachedError, Timeout::Error,
             Mechanize::UnauthorizedError, Mechanize::UnsupportedSchemeError,
             Mechanize::ResponseReadError, Mechanize::ChunkedTerminationError => e
-          raise Relaton::RequestError, "No document found for #{code} reference. #{e.message}"
+          raise Relaton::RequestError, "No document found for #{ref} reference. #{e.message}"
         end
       end
     end

data/lib/relaton/cie.rb

@@ -1,4 +1,4 @@
-require "nokogiri"
+require "moxml"
 # require "parslet"
 require "pubid"
 require "relaton/index"

data/lib/relaton/cloud.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require "lutaml/store"
+
+module Relaton
+  # Access to LutaML data repositories through lutaml-store — the cloud
+  # store (api.relaton.org), a Pages/raw host, or a local GCR-style package,
+  # all through one read API with an explicit local package cache.
+  #
+  # This module composes lutaml-store sources; it is deliberately
+  # independent of Relaton::Db's own cache machinery (see
+  # relaton#204/#205 for that rework).
+  module Cloud
+    module_function
+
+    # An explicitly configured source over a LutaML data repository.
+    #
+    # @param base_url [String] cloud store base (e.g. "https://api.relaton.org")
+    # @param collection [String] one collection (for relaton: a flavor)
+    # @param options [Hash] passthrough to Lutaml::Store::Source::Rest
+    #   (headers:, timeout:, transport:, cache:, default_format:)
+    def source(base_url:, collection:, **options)
+      Lutaml::Store::Source.for(:rest, base_url: base_url, collection: collection, **options)
+    end
+
+    # A local package directory source (a downloaded distribution or a
+    # Mirror/Repository pull).
+    def local(path)
+      Lutaml::Store::Source.for(:directory, path: path)
+    end
+
+    # The record's bytes. NotFoundError means the repository definitively
+    # does not have the key; BackendError means the network/service broke.
+    #
+    # @param key [String] the collection's storage key (see .resolve_key)
+    def read(key, source:)
+      source.read(key)
+    end
+
+    # Resolves a reference to the collection's storage key. The contract's
+    # keys are URL-safe storage keys; slash-bearing docids live in the
+    # manifest's entries[].metadata.docid. Resolution is manifest-driven and
+    # explicit: exact storage-key match first, then the docid metadata,
+    # then nil — there is no path-encoding guesswork.
+    #
+    # @param ref [String] a storage key or a primary docidentifier
+    # @return [String, nil]
+    def resolve_key(ref, source:)
+      manifest = begin
+        source.manifest
+      rescue Lutaml::Store::NotFoundError
+        # a fresh local package holds no manifest yet — it holds no keys
+        return nil
+      end
+
+      return ref if manifest.key?(ref)
+
+      found = manifest.entries.find do |e|
+        e.metadata["docid"]&.casecmp?(ref)
+      end
+      found&.key
+    end
+
+    # The parsed record: the model class is declared by the caller — never
+    # inferred.
+    def get(key, model_class, source:)
+      source.get(key, model_class)
+    end
+
+    # Reference in, parsed model out: resolves the reference through the
+    # manifest, reads the record (from the cloud source, or the local
+    # package when one is given), and deserializes with the caller's model
+    # class. The record's format is self-describing (Format.guess), so the
+    # same call serves YAML, JSON and XML records.
+    #
+    # @param ref [String] a storage key or a primary docidentifier
+    # @param model_class [Class] e.g. Relaton::Bib::Item (YAML/JSON
+    #   records) or Relaton::Bib::Bibdata (XML bibdata records)
+    # @param cache [Lutaml::Store::Source::Directory, nil] read-through
+    #   local package (writes through on a miss)
+    # @return the model instance
+    def fetch(ref, model_class, source:, cache: nil)
+      if cache
+        # Offline-first: resolve and read against the local package before
+        # touching the source — its manifest carries the same docid metadata.
+        local_key = resolve_key(ref, source: cache)
+        if local_key
+          return deserialize(cache.read(local_key), model_class)
+        end
+      end
+
+      key = resolve_key(ref, source: source)
+      raise Lutaml::Store::NotFoundError, "unknown reference: #{ref.inspect}" unless key
+
+      body = if cache
+               Lutaml::Store::Repository.new(source: source, cache: cache).read(key)
+             else
+               source.read(key)
+             end
+
+      deserialize(body, model_class)
+    end
+
+    def deserialize(body, model_class)
+      Lutaml::Store::Format.resolve(Lutaml::Store::Format.guess(body))
+                           .deserialize(body, model_class)
+    end
+
+    # Mirrors the whole collection into a GCR-style local package.
+    #
+    # @param into [String] target directory (a `<into>/<collection>` package
+    #   is written)
+    # @return [Lutaml::Store::Source::Directory] a source over the package
+    def pull(source:, into:, collection: nil, force: false)
+      Lutaml::Store::Mirror.pull(source, into: into, collection: collection, force: force)
+    end
+
+    # Read-through convenience: a Repository over a cloud source with an
+    # explicit local package cache; reads served from the cache once pulled.
+    def repository(base_url:, collection:, cache_path:, **options)
+      Lutaml::Store::Repository.new(
+        source: source(base_url: base_url, collection: collection, **options),
+        cache: local(cache_path)
+      )
+    end
+  end
+end

data/lib/relaton/core/hit_collection.rb

@@ -1,5 +1,5 @@
 require "forwardable"
-require "nokogiri"
+require "moxml"
 require "weakref"
 require_relative "hit"
 
@@ -51,21 +51,17 @@ module Relaton
       # Renders the collection as XML
       #
       # @param opts [Hash] options
-      # @option opts [Nokogiri::XML::Builder] :builder XML builder
+      # @option opts [Moxml::Builder] :builder XML builder
       # @option opts [Boolean] :bibdata render bibdata if true
       # @option opts [String, Symbol] :lang language
       #
       # @return [String] XML representation of the collection
       #
       def to_xml(**opts)
-        builder = Nokogiri::XML::Builder.new(encoding: "UTF-8") do |xml|
-          xml.documents do
-            @array.each do |hit|
-              xml << hit.to_xml(**opts)
-            end
-          end
-        end
-        builder.to_xml
+        parts = @array.map { |hit| hit.to_xml(**opts) }.join
+        Moxml.parse(
+          "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<documents>#{parts}</documents>",
+        ).to_xml(indent: 0, expand_empty: false)
       end
 
       #

data/lib/relaton/core/processor.rb

@@ -32,6 +32,127 @@ module Relaton
                       end
       end
 
+      # The pubid identifier class `Relaton::Db` keys its cache with. Read from
+      # @pubid_identifier, else @pubid_flavor. @pubid_identifier exists so a
+      # flavor can key its cache with pubid without also sourcing #prefixes
+      # from pubid. Nil for a processor with neither: `Db` then keys its cache
+      # with the legacy string key.
+      #
+      # @return [Class, nil]
+      def pubid_class
+        flavor = @pubid_identifier || @pubid_flavor
+        return unless flavor
+
+        require "pubid"
+        @pubid_class ||= ::Pubid.const_get(flavor)::Identifier
+      end
+
+      # Whether the flavor's identifiers print without a publisher token
+      # (OGC `19-025r1`, 3GPP `TS 23.207:REL-18/18.0.0`). Routing then takes
+      # an exact parse by the flavor although no prefix of it claims the
+      # reference (Registry#route).
+      #
+      # @return [Boolean]
+      def bare_identifiers?
+        false
+      end
+
+      # Parse a query reference into a pubid. A flavor that normalizes a
+      # reference before it parses overrides this. A reference the flavor's
+      # grammar cannot read gives no pubid (relaton#235): `get` then
+      # receives the String and normalizes it itself, and the query is not
+      # keyed - the same reading as a flavor's own miss.
+      #
+      # @param ref [String]
+      # @return [Pubid::Identifier, nil]
+      def cache_pubid(ref)
+        pubid_class&.parse(ref)
+      rescue ArgumentError, ::Pubid::Errors::Error, Parslet::ParseFailed
+        # relaton-bib carries the log facade; core specs load without it.
+        require "relaton/bib"
+        Relaton::Bib::Util.info "does not parse as this flavor's pubid; " \
+                                "passing the reference to the flavor as " \
+                                "written", "relaton-db", key: ref
+        nil
+      end
+
+      # The reference as `get` receives it and the cache keys it (relaton#205):
+      # the parse routing already made, else the flavor's own. Nil when the
+      # flavor gives no pubid: `get` then receives the String, and the query
+      # is not cached. A flavor overrides this when an option keeps a query
+      # out of the cache and away from the pubid (CCSDS format).
+      #
+      # @param ref [String]
+      # @param opts [Hash]
+      # @param parsed [Pubid::Identifier, nil] routing's parse, of this
+      #   flavor's class
+      # @return [Pubid::Identifier, nil]
+      def query_pubid(ref, _opts = {}, parsed = nil)
+        pubid = parsed || cache_pubid(ref)
+        # A flavor that reads an unparseable reference as a miss (IANA,
+        # IEEE) gives nil or the raw String: no pubid then. The nil test comes
+        # first: without a pubid class, pubid may not be loaded.
+        pubid if !pubid.nil? && pubid.is_a?(::Pubid::Identifier)
+      end
+
+      # The publishers of a co-published identifier after the lead, in the
+      # order the parsed pubid holds them: the printed order (pubid#472).
+      # Relaton::Db asks their flavors when the lead's catalog has no record
+      # (relaton#205). A flavor whose pubid keeps them elsewhere overrides
+      # this (IEEE).
+      #
+      # @param pubid [Pubid::Identifier] the query pubid (#query_pubid)
+      # @return [Array<String>] publisher abbreviations, e.g. `["IEC"]`
+      def copublishers(pubid)
+        pubid = pubid.identifiers.first if pubid.all_parts?
+        return [] unless pubid.respond_to?(:copublishers)
+
+        Array(pubid.copublishers).map(&:to_s)
+      end
+
+      # Set the year on a parsed pubid, where the flavor's `get` applies it. The
+      # default sets the identifier's own year. A flavor whose `get` applies
+      # the year elsewhere overrides this (see #fold_year_on_root).
+      #
+      # @param pubid [Pubid::Identifier]
+      # @param year [String, Integer]
+      # @return [Pubid::Identifier]
+      def fold_year(pubid, year)
+        pubid.class.from_hash pubid.to_hash.merge("year" => year.to_s)
+      end
+
+      # Set the year on the document the identifier names (`#root`): the base
+      # document of a supplement, the adopted document of an adoption. For a
+      # flavor whose `get` applies the year there (ISO, CEN, BSI).
+      #
+      # @param pubid [Pubid::Identifier]
+      # @param year [String, Integer]
+      # @return [Pubid::Identifier]
+      def fold_year_on_root(pubid, year)
+        copy = pubid.class.from_hash pubid.to_hash
+        copy.root.date = ::Pubid::Components::Date.new(year: year.to_s)
+        copy
+      end
+
+      # The cache key for a query: the parsed pubid with the `year` and
+      # `all_parts` options folded in. The publication date range is not part
+      # of the identity, so it never enters the key.
+      #
+      # @param ref [String]
+      # @param year [String, Integer, nil]
+      # @param opts [Hash]
+      # @param parsed [Pubid::Identifier, nil] the reference as already parsed
+      #   with this flavor's class (#query_pubid); parsed here otherwise
+      # @return [Pubid::Identifier, nil] never the given object itself when a
+      #   year or all-parts is folded in: those fold on copies
+      def cache_key(ref, year, opts, parsed = nil)
+        pubid = query_pubid(ref, opts, parsed) or return
+
+        pubid = fold_year(pubid, year) if year
+        pubid = pubid.to_all_parts if opts[:all_parts] && !pubid.all_parts?
+        pubid
+      end
+
       def initialize
         raise "This is an abstract class!"
       end

data/lib/relaton/core/request_error.rb

@@ -1,12 +1,30 @@
 module Relaton
+  # The base of relaton's own errors. It depends on nothing, so
+  # `relaton/index` requires this file alone and does not load the rest of
+  # `relaton/core`.
+  class Error < StandardError; end
+
   # Transport failure while fetching remote data; `Relaton::Db#net_retry`
-  # retries it. It depends on nothing, so `relaton/index` requires this file
-  # alone and does not load the rest of `relaton/core`.
+  # retries it.
   #
   # The class is named `Relaton::RequestError`, not `Relaton::Core::...`: an
   # error raised without a message uses the class name as its message, and
   # relaton-cli prints that message.
-  class RequestError < StandardError; end
+  class RequestError < Error; end
+
+  # `Relaton::Db` found no flavor for a reference: no flavor's pubid grammar
+  # reads it exactly, and no flavor's prefix matches it. relaton-cli rescues
+  # it, logs the message and returns no document.
+  class UnknownReferenceError < Error
+    # @return [String] the reference no flavor recognizes
+    attr_reader :reference
+
+    # @param reference [String]
+    def initialize(reference)
+      @reference = reference
+      super("`#{reference}` is not a recognized standards identifier")
+    end
+  end
 
   module Core
     RequestError = Relaton::RequestError