relaton 3.0.0.pre.alpha.4 → 3.0.0.pre.alpha.5

data/lib/relaton/db/cache_entry.rb

@@ -0,0 +1,33 @@
+require "lutaml/model"
+
+module Relaton
+  class Db
+    # One row of the cache index. A row is keyed either by a pubid (`id`, its
+    # `to_hash`) or, for a processor with no pubid class, by a string (`key`).
+    # `file` names the document in the doc store; several rows can share one
+    # file (a query row and the row of the document it returned). A
+    # `not_found` row has no file.
+    class CacheEntry < Lutaml::Model::Serializable
+      DOC = "doc".freeze
+      NOT_FOUND = "not_found".freeze
+
+      attribute :id, :hash
+      attribute :key, :string
+      attribute :status, :string
+      attribute :file, :string
+      attribute :fetched, :string
+
+      key_value do
+        map "id", to: :id
+        map "key", to: :key
+        map "status", to: :status
+        map "file", to: :file
+        map "fetched", to: :fetched
+      end
+
+      def not_found?
+        status == NOT_FOUND
+      end
+    end
+  end
+end

data/lib/relaton/db/registry.rb

@@ -92,6 +92,58 @@ module Relaton
       end
 
       #
+      # The processor whose pubid class the identifier belongs to. Only the
+      # generic `Pubid::AllPartsIdentifier`, which belongs to no flavor, is
+      # matched through the document it wraps (`#root`). Any other identifier
+      # is matched by its own class: the `#root` of an adoption is the adopted
+      # document, so `CEN ISO/TS 21003-7` would otherwise be filed as ISO.
+      #
+      # @param pubid [Pubid::Identifier]
+      # @return [Relaton::Core::Processor, nil]
+      #
+      def processor_by_pubid(pubid)
+        generic = pubid.instance_of?(::Pubid::AllPartsIdentifier)
+        id = generic ? pubid.root : pubid
+        processors.values.detect do |processor|
+          klass = processor.pubid_class
+          klass && id.is_a?(klass)
+        end
+      end
+
+      #
+      # Pubid namespaces whose spelling differs from the processor short
+      # name.
+      PUBID_FLAVOR_ALIASES = { "cencenelec" => "cen", "tgpp" => "3gpp" }.freeze
+
+      # Flavors routed by the parsed Pubid class (relaton#205, pilot). Only
+      # processors on this list answer parse-first routing: an allowlist,
+      # not a guess. Pubid::Un is deliberately absent — its Document
+      # identifier parses DOI-shaped strings ("10.17487/RFC3986") that must
+      # not be hijacked away from their current handling. DOI and ISBN are
+      # present: their canonical forms carry their own token (`doi:…`,
+      # `ISBN …`), which pubid detects exactly as the prefix regex routes it.
+      PARSE_ROUTED_FLAVORS = %w[
+        bipm bs cencenelec calconnect cc ccsds cen cie csa doi
+        ecma ecs etsi gost iala iana iec ieee ietf isbn iso itu
+        jcgm jis nist oasis ogc oiml plateau w3c xsf 3gpp
+      ].freeze
+
+      # Find the processor that owns the parsed Pubid's flavor. The
+      # namespace is matched against the processor short name, with the
+      # alias table for the spellings that differ.
+      #
+      # @param pubid [Pubid::Core::Identifier] parsed query
+      # @return [Symbol, nil] standard class name
+      #
+      def class_by_pubid(pubid)
+        ns = pubid.class.name.split("::")[1]&.downcase
+        flavor = PUBID_FLAVOR_ALIASES.fetch(ns, ns)
+        return nil unless PARSE_ROUTED_FLAVORS.include?(flavor)
+
+        key = "relaton_#{flavor}".to_sym
+        processors.key?(key) ? key : nil
+      end
+
       # Find processor by refernce or prefix
       #
       # @param ref [String] reference or prefix

data/lib/relaton/db.rb

@@ -1,11 +1,13 @@
 require "relaton/bib"
 require "yaml"
 require "net/http"
-require "nokogiri"
+require "moxml"
 require "fileutils"
 require "date"
 
 module Relaton
+  autoload :Cloud, "relaton/cloud"
+
   class Db
     # @param global_cache [String] directory of global DB
     # @param local_cache [String] directory of local DB
@@ -64,7 +66,14 @@ module Relaton
     ##
     def fetch(text, year = nil, opts = {})
       reference = text.strip
-      stdclass = @registry.class_by_ref(reference) || return
+      require "pubid"
+      parsed = begin
+        Pubid.parse(reference)
+      rescue Pubid::Errors::Error, Parslet::ParseFailed
+        nil
+      end
+      stdclass = (parsed && @registry.class_by_pubid(parsed)) ||
+                 @registry.class_by_ref(reference) || return
       processor = @registry[stdclass]
       ref = if processor.respond_to?(:urn_to_code)
               processor.urn_to_code(reference)&.first
@@ -90,8 +99,8 @@ module Relaton
       result = []
       db = @db || @local_db
       if db
-        result += db.all do |file, xml|
-          search_xml file, xml, text, edition, year
+        result += db.all do |processor, xml|
+          search_xml processor, xml, text, edition, year
         end.compact
       end
       result
@@ -169,9 +178,10 @@ module Relaton
     # @return [String]
     def to_xml
       db = @local_db || @db || return
-      Nokogiri::XML::Builder.new(encoding: "UTF-8") do |xml|
-        xml.documents { xml.parent.add_child db.all.join(" ") }
-      end.to_xml
+      parts = db.all.join(" ")
+      Moxml.parse(
+        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<documents>#{parts}</documents>",
+      ).to_xml(indent: 0, expand_empty: false)
     end
 
     private
@@ -195,17 +205,15 @@ module Relaton
       opts.merge(code: code, year: year).map { |k, v| "#{k}=#{v}" }.join "&"
     end
 
-    def search_xml(file, xml, text, edition, year)
+    def search_xml(processor, xml, text, edition, year)
+      return unless processor
       return unless text.nil? || match_xml_text?(xml, text)
 
-      search_edition_year(file, xml, edition, year)
+      search_edition_year(processor, xml, edition, year)
     end
 
-    def search_edition_year(file, content, edition, year) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
-      processor = @registry.processor_by_ref(file.split("/")[-2])
-      item = if file.match?(/xml$/) then processor.from_xml(content)
-             else processor.from_yaml(content)
-             end
+    def search_edition_year(processor, content, edition, year) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+      item = processor.from_xml(content)
       item if (edition.nil? || item.edition.content == edition) && (year.nil? ||
         item.date.detect do |d|
           d.type == "published" && d.at.to_date.year.to_s == year.to_s
@@ -240,7 +248,9 @@ module Relaton
       if updates
         doc.relation << Bib::Relation.new(bibitem: updates, type: "updates")
       end
-      divider = stdclass == :relaton_itu ? " " : "/"
+      # The supplement joins its base as the flavor's identifier spells it:
+      # `NIST SP 800-38A Add`, not `/Add`, which pubid does not parse.
+      divider = %i[relaton_itu relaton_nist].include?(stdclass) ? " " : "/"
       refs[1..].each_with_object(doc) do |c, d|
         bib = check_bibliocache(ref + divider + c, year, opts, stdclass)
         if bib
@@ -251,18 +261,52 @@ module Relaton
       end
     end
 
+    # The legacy string cache key, for a processor with no pubid class. The
+    # publication date range is not part of it: it filters, it is not
+    # identity.
     def std_id(code, year, opts, stdclass)
       prefix, code = strip_id_wrapper(code, stdclass)
       ret = code
       ret += (stdclass == :relaton_gb ? "-" : ":") + year if year
       ret += " (all parts)" if opts[:all_parts]
-      after = opts[:publication_date_after]
-      ret += " after-#{after}" if after
-      before = opts[:publication_date_before]
-      ret += " before-#{before}" if before
       ["#{prefix}(#{ret.strip})", code]
     end
 
+    #
+    # The cache key of a query: the flavor's parsed pubid, with the `year`
+    # and `all_parts` options folded in. A reference the flavor cannot parse
+    # raises `Pubid::Errors::ParseError`. Nil when the flavor has a pubid
+    # class but gives no key for this query (a miss by the flavor's own rule,
+    # or a query whose answer the cache cannot hold, such as a CCSDS format):
+    # that query is not cached. A processor with no pubid class gets the
+    # legacy string key.
+    #
+    # @return [Pubid::Identifier, String, nil]
+    #
+    def cache_key(code, year, opts, stdclass)
+      processor = @registry[stdclass]
+      unless processor.pubid_class
+        return std_id(code, year, opts, stdclass).first
+      end
+
+      processor.cache_key(code, year, opts)
+    end
+
+    # The key of a fetched document, from its primary identifier. The
+    # identifier is data, so an unparseable one gives no key.
+    def item_key(bib, stdclass)
+      docid = bib.docidentifier.detect(&:primary) || bib.docidentifier.first
+      return unless docid&.content
+
+      cache_key docid.content, nil, {}, stdclass
+    rescue ::Pubid::Errors::Error, Parslet::ParseFailed
+      nil
+    end
+
+    def date_range?(opts)
+      opts[:publication_date_before] || opts[:publication_date_after]
+    end
+
     def strip_id_wrapper(code, stdclass)
       prefix = @registry[stdclass].prefix
       code =
@@ -280,23 +324,9 @@ module Relaton
     end
 
     def check_bibliocache(code, year, opts, stdclass) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
-      if opts[:publication_date_before] || opts[:publication_date_after]
-        base_opts = opts.except(
-          :publication_date_before, :publication_date_after
-        )
-        base_id, = std_id(code, year, base_opts, stdclass)
-        db = @local_db || @db
-        if db&.valid_entry?(base_id, year)
-          entry = db[base_id]
-          if entry && !entry.match?(/^not_found/) &&
-              pub_date_in_range?(entry, opts)
-            return bib_retval(entry, stdclass)
-          end
-        end
-      end
-
-      id, searchcode = std_id(code, year, opts, stdclass)
-      db = @local_db || @db
+      _, searchcode = strip_id_wrapper(code, stdclass)
+      id = cache_key(searchcode, year, opts, stdclass)
+      db = id && (@local_db || @db)
       altdb = @local_db && @db ? @db : nil
       if db.nil?
         return if opts[:fetch_db]
@@ -304,10 +334,11 @@ module Relaton
         bibentry = new_bib_entry(searchcode, year, opts, stdclass)
         return bib_retval(bibentry, stdclass)
       end
-
-      @semaphore.synchronize do
-        db.delete(id) unless db.valid_entry?(id, year)
+      if date_range?(opts)
+        return check_date_range(searchcode, id, year, opts, stdclass)
       end
+
+      @semaphore.synchronize { db.expire id, year }
       if altdb
         return bib_retval(altdb[id], stdclass) if opts[:fetch_db]
 
@@ -340,28 +371,60 @@ module Relaton
       entry
     end
 
-    def fetch_entry(code, year, opts, stdclass, **args)
+    def fetch_entry(code, year, opts, stdclass, **args) # rubocop:disable Metrics/AbcSize
       processor = @registry[stdclass]
       bib = net_retry(code, year, opts, processor, opts.fetch(:retries, 1))
-
-      entry = check_entry(bib, stdclass, **args)
+      entry = bib_entry bib
       return entry if args[:db].nil?
 
-      @semaphore.synchronize { args[:db][args[:id]] ||= entry }
-      args[:no_cache] ? bib_entry(bib) : entry
+      # `no_cache` refreshes a cached entry, but a failed fetch does not
+      # replace a cached document with `not_found`.
+      refresh = opts[:no_cache] && bib.respond_to?(:to_xml)
+      @semaphore.synchronize do
+        if refresh || !args[:db][args[:id]]
+          save_bib args[:db], args[:id], bib, entry, stdclass
+        end
+      end
+      entry
     end
 
-    def check_entry(bib, stdclass, **args) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
-      bib_id = bib && bib.docidentifier.first&.content
+    #
+    # Cache a fetched document. The document's own identifier gets a row;
+    # when the query key differs from it (an undated or incomplete query),
+    # the query gets a row that points to the same document.
+    #
+    def save_bib(db, key, bib, entry, stdclass)
+      item = bib.respond_to?(:docidentifier) && item_key(bib, stdclass)
+      db.store key, entry, item_key: item || nil
+    end
 
-      quoted = Regexp.quote("(#{bib_id})")
-      if args[:db] && args[:id] && bib_id &&
-          args[:id] !~ /#{quoted}/
-        bid = std_id(bib.docidentifier.first.content, nil, {}, stdclass).first
-        @semaphore.synchronize { args[:db][bid] ||= bib_entry bib }
-        "redirection #{bid}"
-      else bib_entry bib
+    #
+    # A publication date range selects among the cached editions of the
+    # reference, so it is not part of the key. On a miss the flavor is asked
+    # with the range, and its answer is cached under its own identifier only:
+    # the query row keeps pointing to the latest edition.
+    #
+    def check_date_range(code, key, year, opts, stdclass) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+      caches = [@local_db, @db].compact
+      cached = caches.flat_map { |c| c.candidates(key) }.filter_map do |_, xml|
+        date = published_date(xml)
+        [date, xml] if date && pub_date_in_range?(xml, opts)
+      end.max_by(&:first)
+      return bib_retval(cached.last, stdclass) if cached && !opts[:no_cache]
+      return if opts[:fetch_db]
+
+      processor = @registry[stdclass]
+      bib = net_retry(code, year, opts, processor, opts.fetch(:retries, 1))
+      return unless bib.respond_to?(:to_xml)
+
+      entry = bib_entry bib
+      item = item_key(bib, stdclass)
+      if item
+        @semaphore.synchronize do
+          [@local_db, @db].compact.each { |c| c.store item, entry }
+        end
       end
+      bib_retval entry, stdclass
     end
 
     def net_retry(code, year, opts, processor, retries)
@@ -380,19 +443,25 @@ module Relaton
       end
     end
 
-    def pub_date_in_range?(entry, opts) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
-      doc = Nokogiri::XML(entry)
-      date_str = doc.at("//date[@type='published']/on")&.text
-      return false unless date_str
+    # @param entry [String] document XML
+    # @return [Date, nil] the published date
+    def published_date(entry)
+      date_str = Moxml.parse(entry)
+        .at_xpath("//date[@type='published']/on")&.text
+      date_str && parse_pub_date(date_str)
+    end
 
-      date = parse_pub_date(date_str)
+    def pub_date_in_range?(entry, opts) # rubocop:disable Metrics/CyclomaticComplexity
+      date = published_date(entry)
       return false unless date
 
+      # `parse_pub_date`, not `Date.parse`: a bound may be "YYYY" or
+      # "YYYY-MM", which `Date.parse` rejects.
       after = opts[:publication_date_after]
-      return false if after && date < Date.parse(after.to_s)
+      return false if after && date < parse_pub_date(after.to_s)
 
       before = opts[:publication_date_before]
-      return false if before && date >= Date.parse(before.to_s)
+      return false if before && date >= parse_pub_date(before.to_s)
 
       true
     end
@@ -407,18 +476,8 @@ module Relaton
       nil
     end
 
-    def open_cache_biblio(dir) # rubocop:disable Metrics/MethodLength
-      return nil if dir.nil?
-
-      db = Cache.new dir
-
-      Dir["#{dir}/*/"].each do |fdir|
-        next if db.check_version?(fdir)
-
-        FileUtils.rm_rf(fdir, secure: true)
-        Util.info "cache #{fdir}: version is obsolete and cache is cleared."
-      end
-      db
+    def open_cache_biblio(dir)
+      dir && Cache.new(dir)
     end
 
     def process_queue(qwp)

data/lib/relaton/doi/crossref.rb

@@ -10,25 +10,42 @@ module Relaton
       #
       # Get a document by DOI from the CrossRef API.
       #
-      # @param [String] doi The DOI.
+      # @param [String, Pubid::Doi::Identifier] doi The DOI.
       #
       # @return [RelatonBib::BibliographicItem, RelatonIetf::IetfBibliographicItem,
       #   RelatonBipm::BipmBibliographicItem, RelatonIeee::IeeeBibliographicItem,
       #   RelatonNist::NistBibliographicItem] The bibitem.
       #
       def get(doi)
-        Util.info "Fetching from search.crossref.org ...", key: doi
-        id = doi.sub(%r{^doi:}, "")
-        message = get_by_id id
+        key = doi.to_s
+        Util.info "Fetching from search.crossref.org ...", key: key
+        message = get_by_id doi_of(doi)
         if message
-          Util.info "Found: `#{message['DOI']}`", key: doi
+          Util.info "Found: `#{message['DOI']}`", key: key
           Parser.parse message
         else
-          Util.info "Not found.", key: doi
+          Util.info "Not found.", key: key
           nil
         end
       end
 
+      #
+      # The DOI itself (`<prefix>/<suffix>`), as the Crossref API takes it.
+      #
+      # A parsed pubid gives it from its components. A String may carry a
+      # `doi:` scheme (in any case) or a `doi.org` URL: Relaton::Db keys every
+      # form Pubid::Doi reads as one DOI, so all of them must reach the same
+      # DOI here, or a miss on one form is cached for the others.
+      #
+      # @param [String, Pubid::Doi::Identifier] doi
+      # @return [String]
+      #
+      def doi_of(doi)
+        return "#{doi.prefix}/#{doi.suffix}" unless doi.is_a?(String)
+
+        doi.sub(%r{\A(?:doi:|https?://(?:dx\.)?doi\.org/)}i, "")
+      end
+
       #
       # Get a document by DOI from the CrossRef API.
       #

data/lib/relaton/doi/processor.rb

@@ -8,6 +8,7 @@ module Relaton
       def initialize # rubocop:disable Lint/MissingSuper
         @short = :relaton_doi
         @prefix = "DOI"
+        @pubid_identifier = :Doi # Db cache key
         @defaultprefix = %r{^doi:}
         @idtype = "DOI"
       end

data/lib/relaton/easc/processor.rb

@@ -12,6 +12,7 @@ module Relaton
       def initialize
         @short = :relaton_easc
         @prefix = "EASC"
+        @pubid_identifier = :Easc # Db cache key
         # Both Cyrillic and Latin series prefixes route here.
         @defaultprefix = %r{^(?:ПМГ|РМГ|PMG|RMG)\b}
         @idtype = "EASC"

data/lib/relaton/ecma/data_fetcher.rb

@@ -128,7 +128,7 @@ module Relaton
       def to_yaml(bib) = bib.to_yaml
       def to_bibxml(bib) = bib.to_rfcxml
 
-      # @param hit [Nokogiri::XML::Element]
+      # @param hit [Moxml::Element]
       def parse_page(hit) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
         DataParser.new(hit, @errors).parse.each { |item| write_file item }
       end

data/lib/relaton/ecma/data_parser.rb

@@ -6,7 +6,7 @@ module Relaton
       #
       # Initialize parser
       #
-      # @param [Nokogiri::XML::Element] hit document hit
+      # @param [Moxml::Element] hit document hit
       # @param [Hash] errors error tracking hash
       #
       def initialize(hit, errors = {})

data/lib/relaton/ecma/edition_parser.rb

@@ -21,7 +21,7 @@ module Relaton
         docid = @bib[:docidentifier]
         @doc.xpath('//div[@id="main"]/div[1]/div/main/article/div/div/standard/div/ul/li').map do |hit|
           bib = @bib.dup
-          id, ed, bib[:date], vol = edition_id_parts hit.at("./span", "./a").text
+          id, ed, bib[:date], vol = edition_id_parts hit.at_xpath("./span|./a").text
           bib[:source] = edition_source(hit) + edition_translation_source(ed)
           next if ed.nil? || ed.empty?
 
@@ -56,7 +56,7 @@ module Relaton
       end
 
       def edition_source(hit)
-        es = { "src" => hit.at("./a"), "pdf" => hit.at("./span/a") }.map do |type, a|
+        es = { "src" => hit.at_xpath("./a"), "pdf" => hit.at_xpath("./span/a") }.map do |type, a|
           Bib::Uri.new(type: type, content: a[:href]) if a
         end.compact
         @errors[:edition_source] &&= es.empty?

data/lib/relaton/ecma/memento_parser.rb

@@ -5,7 +5,7 @@ module Relaton
 
       ATTRS = %i[docidentifier title date source ext].freeze
 
-      # @param [Nokogiri::XML::Element] hit document hit
+      # @param [Moxml::Element] hit document hit
       # @param [Hash] errors error tracking hash
       def initialize(hit:, errors: {})
         @hit = hit
@@ -23,7 +23,7 @@ module Relaton
 
       # @return [Array<Relaton::Ecma::Docidentifier>]
       def fetch_docidentifier
-        code = "ECMA MEM/#{@hit.at('div[1]//p').text}"
+        code = "ECMA MEM/#{@hit.at_xpath('div[1]//p').text}"
         docid = super(code)
         @errors[:memento_docidentifier] &&= docid.empty?
         docid
@@ -31,7 +31,7 @@ module Relaton
 
       # @return [Array<Relaton::Bib::Title>]
       def fetch_title
-        year = @hit.at("div[1]//p").text
+        year = @hit.at_xpath("div[1]//p").text
         content = "\"Memento #{year}\" for year #{year}"
         result = [Bib::Title.new(content: content, language: "en", script: "Latn")]
         @errors[:memento_title] &&= result.empty?
@@ -40,7 +40,7 @@ module Relaton
 
       # @return [Array<Relaton::Bib::Date>]
       def fetch_date
-        date = @hit.at("div[2]//p").text
+        date = @hit.at_xpath("div[2]//p").text
         on = Date.strptime(date, "%B %Y").strftime "%Y-%m"
         result = [Bib::Date.new(type: "published", at: on)]
         @errors[:memento_date] &&= result.empty?

data/lib/relaton/ecma/standard_parser.rb

@@ -5,7 +5,7 @@ module Relaton
 
       ATTRS = %i[docidentifier title date source abstract relation edition ext].freeze
 
-      # @param [Nokogiri::XML::Element] hit document hit
+      # @param [Moxml::Element] hit document hit
       # @param [Mechanize::Page] doc fetched document page
       # @param [Hash] errors error tracking hash
       def initialize(hit:, doc:, errors: {})
@@ -68,7 +68,7 @@ module Relaton
       def fetch_source # rubocop:disable Metrics/AbcSize
         source = []
         source << Bib::Uri.new(type: "src", content: @hit[:href]) if @hit[:href]
-        ref = @doc.at('//div[@class="ecma-item-content-wrapper"]/span/a',
+        ref = @doc.at_xpath('//div[@class="ecma-item-content-wrapper"]/span/a',
                       '//div[@class="ecma-item-content-wrapper"]/a')
         source << Bib::Uri.new(type: "pdf", content: ref[:href]) if ref
         result = source + edition_translation_source(fetch_edition_content)
@@ -80,7 +80,7 @@ module Relaton
       def fetch_relation # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity
         edition_parser = EditionParser.new(doc: @doc, bib: {}, errors: @errors)
         result = @doc.xpath("//ul[@class='ecma-item-archives']/li").filter_map do |rel|
-          ref, ed, date, vol = edition_parser.edition_id_parts rel.at("span").text
+          ref, ed, date, vol = edition_parser.edition_id_parts rel.at_xpath("span").text
           next if ed.nil? || ed.empty?
 
           docid = Docidentifier.new(type: "ECMA", content: ref, primary: true)
@@ -109,7 +109,7 @@ module Relaton
       private
 
       def fetch_edition_content
-        @doc.at('//p[@class="ecma-item-edition"]')&.text&.match(/^\d+(?=(?:st|nd|th|rd))/)&.to_s
+        @doc.at_xpath('//p[@class="ecma-item-edition"]')&.text&.match(/^\d+(?=(?:st|nd|th|rd))/)&.to_s
       end
 
       def edition_translation_source(edition)
@@ -120,8 +120,8 @@ module Relaton
         return [] unless @doc
 
         @doc.xpath("//h2[.='Translations']/following-sibling::ul/li").map do |l|
-          a = l.at("span/a")
-          id = l.at("span").text
+          a = l.at_xpath("span/a")
+          id = l.at_xpath("span").text
           %r{\w+[\d-]+,\s(?<lang>\w+)\sversion,\s(?<ed>[\d.]+)(?:st|nd|rd|th)\sedition} =~ id
           case lang
           when "Japanese"

data/lib/relaton/etsi/processor.rb

@@ -8,6 +8,7 @@ module Relaton
       def initialize # rubocop:disable Lint/MissingSuper
         @short = :relaton_etsi
         @prefix = "ETSI"
+        @pubid_identifier = :Etsi # Db cache key
         @defaultprefix = %r{^ETSI\s}
         @idtype = "ETSI"
         @datasets = %w[etsi-csv]

data/lib/relaton/gb/gb_scraper.rb

@@ -1,7 +1,7 @@
 # encoding: UTF-8
 # frozen_string_literal: true
 
-require "nokogiri"
+require "moxml"
 require_relative "scraper"
 
 module Relaton
@@ -20,10 +20,10 @@ module Relaton
           hits = doc.xpath(
             "//table[contains(@class, 'result_list')]/tbody[2]/tr",
           ).map do |h|
-            ref = h.at "./td[2]/a"
+            ref = h.at_xpath "./td[2]/a"
             pid = ref[:onclick].match(/[0-9A-F]+/).to_s
-            status = h.at("./td[7]").text.strip
-            rdate = h.at("./td[8]").text.strip
+            status = h.at_xpath("./td[7]").text.strip
+            rdate = h.at_xpath("./td[8]").text.strip
             Hit.new pid: pid, docref: ref.text, scraper: self,
                     release_date: rdate, status: status
           end
@@ -52,7 +52,7 @@ module Relaton
         #   * :type [String]
         #   * :name [String]
         # def get_committee(doc, _ref)
-        #   name = doc.at("//div[contains(., '归口单位') or contains(., '归口部门')]/following-sibling::div")
+        #   name = doc.at_xpath("//div[contains(., '归口单位') or contains(., '归口部门')]/following-sibling::div")
         #   Committee.new(type: "technical", content: name.text.delete("\r\n\t\t"))
         # end
       end