relaton 3.0.0.pre.alpha.4 → 3.0.0.pre.alpha.5

data/lib/relaton/cen/scraper.rb

@@ -53,7 +53,7 @@ module Relaton
         # @param doc [Mechanize::Page]
         # @return [Array<Relaton::Bib::LocalizedMarkedUpString>]
         def fetch_abstract(doc)
-          content = doc.at("//tr[th[.='Abstract/Scope']]/td")
+          content = doc.at_xpath("//tr[th[.='Abstract/Scope']]/td")
           [Bib::Abstract.new(content: content.text, language: "en", script: "Latn")]
         end
 
@@ -68,7 +68,7 @@ module Relaton
         # @param doc [Mechanize::Page]
         # @return [Relaton::Bib::DocumentStatus, nil]
         def fetch_status(doc)
-          s = doc.at("//tr[th[.='Status']]/td")
+          s = doc.at_xpath("//tr[th[.='Status']]/td")
           return unless s
 
           stage = Bib::Status::Stage.new(content: s.text.strip)
@@ -98,7 +98,7 @@ module Relaton
           doc.xpath(
             "//div[@id='DASHBOARD_LISTRELATIONS']/table/tr[th[.!='Sales Points']]",
           ).each_with_object([]) do |rt, acc|
-            type = relation_type rt.at("th").text.downcase
+            type = relation_type rt.at_xpath("th").text.downcase
             rt.xpath("td/a").each do |r|
               acc << Bib::Relation.new(type: type, bibitem: create_relation(r))
             end
@@ -124,7 +124,7 @@ module Relaton
         # @param doc [Mechanize::Page]
         # @return [Array<Relaton::Bib::Title>]
         def fetch_titles(doc)
-          te = doc.at("//tr[th[.='Title']]/td").text.strip
+          te = doc.at_xpath("//tr[th[.='Title']]/td").text.strip
           Bib::Title.from_string te, "en", "Latn"
         end
 
@@ -134,10 +134,10 @@ module Relaton
         def fetch_dates(doc) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/MethodLength
           doc.xpath("//div[@id='DASHBOARD_LISTIMPLEMENTATIONDATES']/table/tr")
             .each_with_object([]) do |d, acc|
-            at = d.at("td").text
+            at = d.at_xpath("td").text
             next if at.empty?
 
-            t = d.at("th").text
+            t = d.at_xpath("th").text
             type = case t
                   when /DOR/ then "adapted"
                   when /DAV/ then "issued"
@@ -154,8 +154,8 @@ module Relaton
         # @param doc [Mechanize::Page]
         # @return [Array<Relaton::Bib::Contributor>]
         def fetch_contributors(doc)
-          code = doc.at("//tr/td/h1/text()").text
-          title = doc.at("//tr/td[3]/h1").text
+          code = doc.at_xpath("//tr/td/h1/text()").text
+          title = doc.at_xpath("//tr/td[3]/h1").text
           %r{/(?<type>\w+)(?:\s(?<num>[^/]+))?$} =~ code
           org = owner_entity
 
@@ -190,7 +190,7 @@ module Relaton
         # @param doc [Mechanize::Page]
         # @return [Array<Bib::Copyright>]
         def fetch_copyright(doc)
-          date = doc.at("//tr[th[.='date of Availability (DAV)']]/td").text
+          date = doc.at_xpath("//tr[th[.='date of Availability (DAV)']]/td").text
           owner = Bib::ContributionInfo.new(organization: owner_entity)
           from = date.match(/^\d{4}/).to_s
           [Bib::Copyright.new(owner: [owner], from: from)]

data/lib/relaton/cie/data_fetcher.rb

@@ -3,7 +3,7 @@
 require "English"
 require "fileutils"
 require "ferrum"
-require "nokogiri"
+require "moxml"
 require "pubid"
 require "relaton/index"
 require "relaton/bib"
@@ -58,7 +58,7 @@ module Relaton
       def get(url)
         @browser.go_to(url)
         wait_for_challenge
-        Nokogiri::HTML(@browser.body)
+        Moxml.new.parse_html(@browser.body)
       end
 
       def quit
@@ -181,7 +181,7 @@ module Relaton
         Util.error msg
       end
 
-      # @param hit [Nokogiri::HTML::Document]
+      # @param hit [Moxml::Document]
       # @param doc [Mechanize::Page]
       # @return [Array<Relaton::Bib::Docidentifier>]
       def fetch_docid(hit, doc)
@@ -200,7 +200,7 @@ module Relaton
         else
           @errors[:docid_2] &&= true
         end
-        isbn = doc.at('//h3[contains(.,"ISBN")]/following-sibling::span')&.text
+        isbn = doc.at_xpath('//h3[contains(.,"ISBN")]/following-sibling::span')&.text
         if isbn && !isbn.strip.empty?
           docid << Bib::Docidentifier.new(type: "ISBN", content: isbn)
           @errors[:docid_isbn] &&= false
@@ -211,7 +211,7 @@ module Relaton
       end
 
       def parse_code(hit, doc = nil)
-        code = hit.at("h3/a").text.strip.squeeze(" ").sub(/\u25b9/, "").gsub(" / ", "/")
+        code = hit.at_xpath(".//h3/a").text.strip.squeeze(" ").sub(/\u25b9/, "").gsub(" / ", "/")
         c2idx = %r{(?:\(|/)(?<c2>(?:ISO|IEC)\s[^()]+)} =~ code
         code = code[0...c2idx].strip if c2idx
         [primary_code(code, doc), c2]
@@ -221,7 +221,7 @@ module Relaton
         /^(?<code1>[^(]+)(?:\((?<code2>[a-zA-Z]+\d+,(?:\sPages)?[^)]+))?/ =~ code
         if code1&.match?(/^CIE/)
           parse_cie_code code1, code2, doc
-        elsif (pcode = doc&.at('//h3[.="Product Code(s):"]/following-sibling::span'))
+        elsif (pcode = doc&.at_xpath('//h3[.="Product Code(s):"]/following-sibling::span'))
           "CIE #{pcode.text.strip.match(/[^,]+/)}"
         else
           num = code.match(/(?<=\()\w{2}\d+,.+(?=\))/).to_s.gsub(/,(?=\s)/, "").gsub(/,(?=\S)/, " ")
@@ -231,7 +231,7 @@ module Relaton
 
       def parse_cie_code(code1, code2, doc = nil) # rubocop:disable Metrics/CyclomaticComplexity
         code = code1.size > 25 && code2 ? "CIE #{code2.sub(/,(\sPages)?/, '')}" : code1
-        add = doc&.at("//hgroup/h2")&.text&.match(/(Add)endum\s(\d+)$/)
+        add = doc&.at_xpath("//hgroup/h2")&.text&.match(/(Add)endum\s(\d+)$/)
         return code unless add
 
         "#{code} #{add[1]} #{add[2]}"
@@ -244,7 +244,7 @@ module Relaton
       # @param doc [Mechanize::Page]
       # @return [Array<Relaton::Bib::Title>]
       def fetch_title(doc)
-        t = doc.at("//hgroup/h2/text()", "//hgroup/h1/text()")
+        t = doc.at_xpath("//hgroup/h2/text()|//hgroup/h1/text()")
         unless t && !t.text.strip.empty?
           @errors[:title] &&= true
           return []
@@ -270,7 +270,7 @@ module Relaton
       # @param doc [Mechanize::Page]
       # @return [String]
       def fetch_edition(doc)
-        ed = doc.at("//h3[.='Edition:']/following-sibling::span")
+        ed = doc.at_xpath("//h3[.='Edition:']/following-sibling::span")
         @errors[:edition] &&= true
         return unless ed
 
@@ -285,16 +285,16 @@ module Relaton
       # @return [Array<Relaton::Cie::Relation>]
       def fetch_relation(doc) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
         rels = doc.xpath('//section[@class="history"]/ol/li[not(contains(@class,"selected-product"))]').map do |rel|
-          ref = rel.at("a")
+          ref = rel.at_xpath(".//a")
           url = "https://www.techstreet.com#{ref[:href]}"
-          title = Bib::Title.from_string ref.at('p/span[@class="title"]').text
-          did = ref.at("h3").text
+          title = Bib::Title.from_string ref.at_xpath('.//p/span[@class="title"]').text
+          did = ref.at_xpath(".//h3").text
           docid = [Bib::Docidentifier.new(type: "CIE", content: did, primary: true)]
-          on = ref.at("p/time")
+          on = ref.at_xpath(".//p/time")
           date = [Bib::Date.new(type: "published", at: on[:datetime])]
           source = [Bib::Uri.new(type: "src", content: url)]
           bibitem = ItemData.new docidentifier: docid, title: title, source: source, date: date
-          type = ref.at('//li/i[contains(@class,"historical")]') ? "updates" : "updatedBy"
+          type = ref.at_xpath('//li/i[contains(@class,"historical")]') ? "updates" : "updatedBy"
           Bib::Relation.new(type: type, bibitem: bibitem)
         end
         @errors[:relation] &&= rels.empty?
@@ -313,7 +313,7 @@ module Relaton
       # @param doc [Mechanize::Page]
       # @return [Array<Relaton::Bib::LocalizedMarkedUpString>]
       def fetch_abstract(doc)
-        content = doc.at('//div[contains(@class,"description")]')&.text&.strip
+        content = doc.at_xpath('//div[contains(@class,"description")]')&.text&.strip
         if content.nil? || content.empty?
           @errors[:abstract] &&= true
           return []
@@ -422,10 +422,10 @@ module Relaton
       # workers. Called directly (single stubbed agent) by specs and from the
       # pool with a per-worker agent/pacing/position.
       #
-      # @param hit [Nokogiri::HTML::Element]
+      # @param hit [Moxml::Element]
       # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       def parse_page(hit, pos = nil, worker_agent = agent, pacing = nil)
-        url = hit.at('h3/a')[:href]
+        url = hit.at_xpath('.//h3/a')[:href]
         doc = time_req(pacing) { worker_agent.get url }
         @mutex.synchronize do
           item = ItemData.new(
@@ -460,7 +460,7 @@ module Relaton
       def collect_hits(url = URL, hits = [])
         result = time_req(listing_pacing) { agent.get url }
         hits.concat result.xpath("//li[@data-product]").to_a
-        np = result.at '//a[@class="next_page"]'
+        np = result.at_xpath '//a[@class="next_page"]'
         return hits unless np
 
         next_href = np[:href]

data/lib/relaton/cie/processor.rb

@@ -6,6 +6,7 @@ module Relaton
       def initialize # rubocop:disable Lint/MissingSuper
         @short = :relaton_cie
         @prefix = "CIE"
+        @pubid_identifier = :Cie # Db cache key
         @defaultprefix = /^CIE(-|\s)/
         @idtype = "CIE"
         @datasets = %w[cie-techstreet]

data/lib/relaton/cie.rb

@@ -1,4 +1,4 @@
-require "nokogiri"
+require "moxml"
 # require "parslet"
 require "pubid"
 require "relaton/index"

data/lib/relaton/cloud.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require "lutaml/store"
+
+module Relaton
+  # Access to LutaML data repositories through lutaml-store — the cloud
+  # store (api.relaton.org), a Pages/raw host, or a local GCR-style package,
+  # all through one read API with an explicit local package cache.
+  #
+  # This module composes lutaml-store sources; it is deliberately
+  # independent of Relaton::Db's own cache machinery (see
+  # relaton#204/#205 for that rework).
+  module Cloud
+    module_function
+
+    # An explicitly configured source over a LutaML data repository.
+    #
+    # @param base_url [String] cloud store base (e.g. "https://api.relaton.org")
+    # @param collection [String] one collection (for relaton: a flavor)
+    # @param options [Hash] passthrough to Lutaml::Store::Source::Rest
+    #   (headers:, timeout:, transport:, cache:, default_format:)
+    def source(base_url:, collection:, **options)
+      Lutaml::Store::Source.for(:rest, base_url: base_url, collection: collection, **options)
+    end
+
+    # A local package directory source (a downloaded distribution or a
+    # Mirror/Repository pull).
+    def local(path)
+      Lutaml::Store::Source.for(:directory, path: path)
+    end
+
+    # The record's bytes. NotFoundError means the repository definitively
+    # does not have the key; BackendError means the network/service broke.
+    #
+    # @param key [String] the collection's storage key (see .resolve_key)
+    def read(key, source:)
+      source.read(key)
+    end
+
+    # Resolves a reference to the collection's storage key. The contract's
+    # keys are URL-safe storage keys; slash-bearing docids live in the
+    # manifest's entries[].metadata.docid. Resolution is manifest-driven and
+    # explicit: exact storage-key match first, then the docid metadata,
+    # then nil — there is no path-encoding guesswork.
+    #
+    # @param ref [String] a storage key or a primary docidentifier
+    # @return [String, nil]
+    def resolve_key(ref, source:)
+      manifest = begin
+        source.manifest
+      rescue Lutaml::Store::NotFoundError
+        # a fresh local package holds no manifest yet — it holds no keys
+        return nil
+      end
+
+      return ref if manifest.key?(ref)
+
+      found = manifest.entries.find do |e|
+        e.metadata["docid"]&.casecmp?(ref)
+      end
+      found&.key
+    end
+
+    # The parsed record: the model class is declared by the caller — never
+    # inferred.
+    def get(key, model_class, source:)
+      source.get(key, model_class)
+    end
+
+    # Reference in, parsed model out: resolves the reference through the
+    # manifest, reads the record (from the cloud source, or the local
+    # package when one is given), and deserializes with the caller's model
+    # class. The record's format is self-describing (Format.guess), so the
+    # same call serves YAML, JSON and XML records.
+    #
+    # @param ref [String] a storage key or a primary docidentifier
+    # @param model_class [Class] e.g. Relaton::Bib::Item (YAML/JSON
+    #   records) or Relaton::Bib::Bibdata (XML bibdata records)
+    # @param cache [Lutaml::Store::Source::Directory, nil] read-through
+    #   local package (writes through on a miss)
+    # @return the model instance
+    def fetch(ref, model_class, source:, cache: nil)
+      if cache
+        # Offline-first: resolve and read against the local package before
+        # touching the source — its manifest carries the same docid metadata.
+        local_key = resolve_key(ref, source: cache)
+        if local_key
+          return deserialize(cache.read(local_key), model_class)
+        end
+      end
+
+      key = resolve_key(ref, source: source)
+      raise Lutaml::Store::NotFoundError, "unknown reference: #{ref.inspect}" unless key
+
+      body = if cache
+               Lutaml::Store::Repository.new(source: source, cache: cache).read(key)
+             else
+               source.read(key)
+             end
+
+      deserialize(body, model_class)
+    end
+
+    def deserialize(body, model_class)
+      Lutaml::Store::Format.resolve(Lutaml::Store::Format.guess(body))
+                           .deserialize(body, model_class)
+    end
+
+    # Mirrors the whole collection into a GCR-style local package.
+    #
+    # @param into [String] target directory (a `<into>/<collection>` package
+    #   is written)
+    # @return [Lutaml::Store::Source::Directory] a source over the package
+    def pull(source:, into:, collection: nil, force: false)
+      Lutaml::Store::Mirror.pull(source, into: into, collection: collection, force: force)
+    end
+
+    # Read-through convenience: a Repository over a cloud source with an
+    # explicit local package cache; reads served from the cache once pulled.
+    def repository(base_url:, collection:, cache_path:, **options)
+      Lutaml::Store::Repository.new(
+        source: source(base_url: base_url, collection: collection, **options),
+        cache: local(cache_path)
+      )
+    end
+  end
+end

data/lib/relaton/core/hit_collection.rb

@@ -1,5 +1,5 @@
 require "forwardable"
-require "nokogiri"
+require "moxml"
 require "weakref"
 require_relative "hit"
 
@@ -51,21 +51,17 @@ module Relaton
       # Renders the collection as XML
       #
       # @param opts [Hash] options
-      # @option opts [Nokogiri::XML::Builder] :builder XML builder
+      # @option opts [Moxml::Builder] :builder XML builder
       # @option opts [Boolean] :bibdata render bibdata if true
       # @option opts [String, Symbol] :lang language
       #
       # @return [String] XML representation of the collection
       #
       def to_xml(**opts)
-        builder = Nokogiri::XML::Builder.new(encoding: "UTF-8") do |xml|
-          xml.documents do
-            @array.each do |hit|
-              xml << hit.to_xml(**opts)
-            end
-          end
-        end
-        builder.to_xml
+        parts = @array.map { |hit| hit.to_xml(**opts) }.join
+        Moxml.parse(
+          "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<documents>#{parts}</documents>",
+        ).to_xml(indent: 0, expand_empty: false)
       end
 
       #

data/lib/relaton/core/processor.rb

@@ -32,6 +32,74 @@ module Relaton
                       end
       end
 
+      # The pubid identifier class `Relaton::Db` keys its cache with. Read from
+      # @pubid_identifier, else @pubid_flavor. @pubid_identifier exists so a
+      # flavor can key its cache with pubid without also sourcing #prefixes
+      # from pubid. Nil for a processor with neither: `Db` then keys its cache
+      # with the legacy string key.
+      #
+      # @return [Class, nil]
+      def pubid_class
+        flavor = @pubid_identifier || @pubid_flavor
+        return unless flavor
+
+        require "pubid"
+        @pubid_class ||= ::Pubid.const_get(flavor)::Identifier
+      end
+
+      # Parse a query reference into a pubid. A flavor that normalizes a
+      # reference before it parses overrides this. A parse error propagates:
+      # an unrecognized reference is not "not found".
+      #
+      # @param ref [String]
+      # @return [Pubid::Identifier, nil]
+      def cache_pubid(ref)
+        pubid_class&.parse(ref)
+      end
+
+      # Set the year on a parsed pubid, where the flavor's `get` applies it. The
+      # default sets the identifier's own year. A flavor whose `get` applies
+      # the year elsewhere overrides this (see #fold_year_on_root).
+      #
+      # @param pubid [Pubid::Identifier]
+      # @param year [String, Integer]
+      # @return [Pubid::Identifier]
+      def fold_year(pubid, year)
+        pubid.class.from_hash pubid.to_hash.merge("year" => year.to_s)
+      end
+
+      # Set the year on the document the identifier names (`#root`): the base
+      # document of a supplement, the adopted document of an adoption. For a
+      # flavor whose `get` applies the year there (ISO, CEN, BSI).
+      #
+      # @param pubid [Pubid::Identifier]
+      # @param year [String, Integer]
+      # @return [Pubid::Identifier]
+      def fold_year_on_root(pubid, year)
+        copy = pubid.class.from_hash pubid.to_hash
+        copy.root.date = ::Pubid::Components::Date.new(year: year.to_s)
+        copy
+      end
+
+      # The cache key for a query: the parsed pubid with the `year` and
+      # `all_parts` options folded in. The publication date range is not part
+      # of the identity, so it never enters the key.
+      #
+      # @param ref [String]
+      # @param year [String, Integer, nil]
+      # @param opts [Hash]
+      # @return [Pubid::Identifier, nil]
+      def cache_key(ref, year, opts)
+        pubid = cache_pubid(ref)
+        # A flavor that reads an unparseable reference as a miss (IANA, IEEE)
+        # gives nil or the raw String: no pubid key then.
+        return if pubid.nil? || !pubid.is_a?(::Pubid::Identifier)
+
+        pubid = fold_year(pubid, year) if year
+        pubid = pubid.to_all_parts if opts[:all_parts] && !pubid.all_parts?
+        pubid
+      end
+
       def initialize
         raise "This is an abstract class!"
       end