relaton 3.0.0.pre.alpha.4 → 3.0.0.pre.alpha.6

data/lib/relaton/isbn/open_library.rb

@@ -8,23 +8,27 @@ module Relaton
 
       ENDPOINT = "http://openlibrary.org/api/volumes/brief/isbn/".freeze
 
-      def get(ref, _date = nil, _opts = {}) # rubocop:disable Metrics/MethodLength
-        Util.info "Fetching from OpenLibrary ...", key: ref
-
-        isbn = Isbn.new(ref).parse
+      # @param ref [String, Pubid::Isbn::Identifier] an ISBN-10 or ISBN-13
+      # @return [Relaton::Bib::ItemData, nil]
+      def get(ref, _date = nil, _opts = {}) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+        Util.info "Fetching from OpenLibrary ...", key: ref.to_s
+
+        # A parsed pubid gives its digits (`raw`); `Isbn#parse` validates them
+        # and converts an ISBN-10 to ISBN-13, as for a String.
+        isbn = Isbn.new(ref.is_a?(String) ? ref : ref.raw).parse
         unless isbn
-          Util.info "Incorrect ISBN.", key: ref
+          Util.info "Incorrect ISBN.", key: ref.to_s
           return
         end
 
         resp = request_api isbn
         unless resp
-          Util.info "Not found.", key: ref
+          Util.info "Not found.", key: ref.to_s
           return
         end
 
         bib = Parser.parse resp
-        Util.info "Found: `#{bib.docidentifier.first.content}`", key: ref
+        Util.info "Found: `#{bib.docidentifier.first.content}`", key: ref.to_s
         bib
       end
 

data/lib/relaton/isbn/processor.rb

@@ -8,6 +8,7 @@ module Relaton
       def initialize # rubocop:disable Lint/MissingSuper
         @short = :relaton_isbn
         @prefix = "ISBN"
+        @pubid_identifier = :Isbn # Db cache key
         @defaultprefix = /^ISBN\s/
         @idtype = "ISBN"
         @datasets = %w[]
@@ -22,6 +23,15 @@ module Relaton
         ::Relaton::Isbn::OpenLibrary.get(code, date, opts)
       end
 
+      # `OpenLibrary.get` reads an incorrect ISBN as a miss, so a reference
+      # pubid cannot parse (it checks the check digit too) gets no key here
+      # either: it is not cached.
+      def cache_pubid(ref)
+        super
+      rescue StandardError
+        nil
+      end
+
       #
       # @param xml [String]
       # @return [Relaton::Bib::Bibitem]

data/lib/relaton/iso/bibliography.rb

@@ -24,7 +24,9 @@ module Relaton
         raise Relaton::RequestError, e.message
       end
 
-      # @param ref [String] the ISO standard Code to look up (e..g "ISO 9000")
+      # @param ref [String, Pubid::Iso::Identifier] the ISO standard Code to
+      #   look up (e..g "ISO 9000"), or the parse that Relaton::Db routed with
+      #   (relaton#205); a pubid is never mutated
       # @param year [String, NilClass] the year the standard was published
       # @param opts [Hash] options; restricted to :all_parts if all-parts
       # @option opts [Boolean] :all_parts if all-parts reference is required
@@ -33,21 +35,21 @@ module Relaton
       #
       # @return [RelatonIsoBib::IsoBibliographicItem] Bibliographic item
       def get(ref, year = nil, opts = {}) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity,Metrics/AbcSize
-        code = ref.gsub("\u2013", "-")
-
-        # parse "all parts" request
-        # code.sub! " (all parts)", ""
-        # opts[:all_parts] ||= $~ && opts[:all_parts].nil?
-
-        query_pubid = ::Pubid::Iso::Identifier.parse(code)
+        query_pubid = if ref.is_a?(String)
+                        ::Pubid::Iso::Identifier.parse(ref.gsub("\u2013", "-"))
+                      else
+                        ref
+                      end
         if year&.respond_to?(:to_i)
+          # Copy first: a caller's pubid is also its Db cache key.
+          query_pubid = query_pubid.class.from_hash(query_pubid.to_hash)
           query_pubid.root.date = ::Pubid::Components::Date.new(year: year.to_s)
         end
         query_pubid = query_pubid.to_all_parts if opts[:all_parts]
         Util.info "Fetching from Relaton repository ...", key: query_pubid.to_s
 
         hits, missed_year_ids = isobib_search_filter(query_pubid, opts)
-        tip_ids = look_up_with_any_types_stages(hits, ref, opts)
+        tip_ids = look_up_with_any_types_stages(hits, ref.to_s, opts)
 
         date_filter = opts[:publication_date_before] || opts[:publication_date_after]
         if date_filter && !query_pubid.all_parts

data/lib/relaton/iso/data_parser.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "nokogiri"
+require "moxml"
 require_relative "../iso"
 require_relative "scraper"
 
@@ -331,7 +331,7 @@ module Relaton
       end
 
       def strip_html(html)
-        Nokogiri::HTML.fragment(html).text.strip.gsub(/\s+/, " ")
+        Moxml.new.parse_fragment(html).map(&:text).join.strip.gsub(/\s+/, " ")
       end
 
       # ---- copyright -------------------------------------------------------

data/lib/relaton/iso/processor.rb

@@ -39,6 +39,11 @@ module Relaton
         DataFetcher.fetch(source, **opts)
       end
 
+      # `Bibliography.get` applies a year to the document the reference names.
+      def fold_year(pubid, year)
+        fold_year_on_root pubid, year
+      end
+
       # @param xml [String]
       # @return [RelatonIsoBib::IsoBibliographicItem]
       def from_xml(xml)

data/lib/relaton/iso/scraper.rb

@@ -95,7 +95,7 @@ module Relaton
       def id
         return @id if defined?(@id)
 
-        did = @doc.at("//h1/span[1]")
+        did = @doc.at_xpath("//h1/span[1]")
         @errors[:id] &&= did.nil?
         @id = did && did.text.split(" | ").first.strip
       end
@@ -117,7 +117,7 @@ module Relaton
       def edition
         return @edition if defined?(@edition)
 
-        ed = @doc.at("//div[div[.='Edition']]/text()[last()]")
+        ed = @doc.at_xpath("//div[div[.='Edition']]/text()[last()]")
         @errors[:edition] &&= ed.nil?
         @edition = ed && Bib::Edition.new(content: ed.text.match(/\d+$/).to_s)
       end
@@ -172,7 +172,7 @@ module Relaton
         langs = languages.each_with_object([]) do |l, s|
           # Don't need to get page for en. We already have it.
           d = l[:path] ? get_page(l[:path])[0] : @doc
-          unless d.at("//h5[@class='help-block'][.='недоступно на русском языке']")
+          unless d.at_xpath("//h5[@class='help-block'][.='недоступно на русском языке']")
             s << l
             titles += fetch_title(d, l[:lang])
 
@@ -208,7 +208,7 @@ module Relaton
 
       # Get page.
       # @param path [String] page's path
-      # @return [Array<Nokogiri::HTML::Document, String>]
+      # @return [Array<Moxml::Document, String>]
       def get_page(path) # rubocop:disable Metrics/MethodLength
         try = 0
         begin
@@ -276,12 +276,12 @@ module Relaton
       # @param [Net::HTTPOK] resp HTTP response
       # @param [URI::HTTPS] uri URI of the page
       #
-      # @return [Nokogiri::HTML4::Document] document
+      # @return [Moxml::Document] document
       # @raise [Relaton::RequestError] if the page could not be parsed
       #
       def try_if_fail(resp, uri)
         10.times do
-          doc = Nokogiri::HTML(resp.body)
+          doc = Moxml.new.parse_html(resp.body)
           # stop trying if page has a document id
           return doc if item_ref(doc)
 
@@ -312,12 +312,12 @@ module Relaton
       #
       # Parse ID from the document.
       #
-      # @param [Nokogiri::HTML::Document] doc document to parse
+      # @param [Moxml::Document] doc document to parse
       #
       # @return [String, nil] ID
       #
       def item_ref(doc)
-        ref = doc.at("//main//section/div/div/div//h1/span[1]")
+        ref = doc.at_xpath("//main//section/div/div/div//h1/span[1]")
         @errors[:reference] &&= ref.nil?
         ref&.text&.strip
       end
@@ -334,7 +334,7 @@ module Relaton
       def stage_code
         return @stage_code if defined?(@stage_code)
 
-        stc = @doc.at("//ul[@class='dropdown-menu']/li[@class='active']/a/span[@class='stage-code']")
+        stc = @doc.at_xpath("//ul[@class='dropdown-menu']/li[@class='active']/a/span[@class='stage-code']")
         @errors[:stage] &&= stc.nil?
         @stage_code = stc&.text
       end
@@ -351,7 +351,7 @@ module Relaton
         rels = @doc.xpath(
           "//ul[@class='steps']/li", "//div[contains(@class, 'sub-step')]"
         ).reduce([]) do |a, r|
-          type, date = relation_type(r.at("h4", "h5").text.strip)
+          type, date = relation_type(r.at_xpath("h4|h5").text.strip)
           next a if types.include?(type)
 
           a + create_relations(r, type, date)
@@ -383,7 +383,7 @@ module Relaton
       #
       # Create relations.
       #
-      # @param [Nokogiri::HTML::Element] rel relation element
+      # @param [Moxml::Element] rel relation element
       # @param [String] type relation type
       # @param [Hash{Symbol=>String}] date relation document date
       # @option date [String] :type date type
@@ -400,7 +400,7 @@ module Relaton
       end
 
       # Fetch titles.
-      # @param doc [Nokogiri::HTML::Document]
+      # @param doc [Moxml::Document]
       # @param lang [String]
       # @return [Array<RelatonBib::TypedTitleString>]
       def fetch_title(doc, lang) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
@@ -417,7 +417,7 @@ module Relaton
       end
 
       def parse_titles(doc)
-        # head = doc.at "//nav[contains(@class,'heading-condensed')]"
+        # head = doc.at_xpath "//nav[contains(@class,'heading-condensed')]"
         ttls = doc.xpath("//h1[@class='stdTitle']/span[position()>1]").map(&:text)
         return ttls if @errors[:title] &&= ttls.empty?
 
@@ -455,14 +455,14 @@ module Relaton
       def fetch_dates # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
         dates = []
         %r{^[^\s]+\s[\d-]+:(?<ref_date_str>\d{4})} =~ id
-        pub_date_str = @doc.at("//span[@itemprop='releaseDate']")
+        pub_date_str = @doc.at_xpath("//span[@itemprop='releaseDate']")
         @errors[:date_pub] &&= pub_date_str.nil?
         if ref_date_str
           dates += parse_date_from_id ref_date_str, pub_date_str
         elsif pub_date_str
           dates << Bib::Date.new(type: "published", at: pub_date_str.text)
         end
-        corr_data = @doc.at "//span[@itemprop='dateModified']"
+        corr_data = @doc.at_xpath "//span[@itemprop='dateModified']"
         @errors[:date_corr] &&= corr_data.nil?
         dates << Bib::Date.new(type: "corrected", at: corr_data.text) if corr_data
         dates
@@ -522,13 +522,13 @@ module Relaton
       #
       def fetch_source(url) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength
         source = [Bib::Uri.new(type: "src", content: url)]
-        obp = @doc.at("//a[.='Read sample']")
+        obp = @doc.at_xpath("//a[.='Read sample']")
         @errors[:link_obp] &&= obp.nil?
         source << Bib::Uri.new(type: "obp", content: obp[:href]) if obp
-        rss = @doc.at("//a[contains(@href, 'rss')]")
+        rss = @doc.at_xpath("//a[contains(@href, 'rss')]")
         @errors[:link_rss] &&= rss.nil?
         source << Bib::Uri.new(type: "rss", content: DOMAIN + rss[:href]) if rss
-        pub = @doc.at "//p[contains(., 'publicly available')]/a",
+        pub = @doc.at_xpath "//p[contains(., 'publicly available')]/a",
                       "//p[contains(., 'can be downloaded from the')]/a"
         @errors[:link_pub] &&= pub.nil?
         source << Bib::Uri.new(type: "pub", content: pub[:href]) if pub
@@ -542,7 +542,7 @@ module Relaton
         owner_name = ref.match(/.*?(?=\s)/).to_s
         from = ref.match(/(?<=:)\d{4}/).to_s
         if from.empty?
-          date = @doc.at(
+          date = @doc.at_xpath(
             "//span[@itemprop='releaseDate']",
             "//ul[@id='stages']/li[contains(@class,'active')]/ul/li[@class='active']/a/span[@class='stage-date']",
           )
@@ -585,7 +585,7 @@ module Relaton
       # @return [Relaton::Bib::Contributor, nil]
       #
       def fetch_editorialgroup_contributor # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
-        wg = @doc.at(
+        wg = @doc.at_xpath(
           "//div[contains(., 'Technical Committe')]" \
           "/following-sibling::span/a",
         )

data/lib/relaton/itu/bibliography.rb

@@ -5,35 +5,97 @@ module Relaton
     module Bibliography
       extend self
 
-      # @param refid [Relaton::Itu::Pubid, String] a document reference
+      # Supplements that print the date of the recommendation they amend
+      # first, "ITU-T G.989.2 (2014) Amd. 1 (04/2016)": their edition is the
+      # base's.
+      AMENDING = [
+        ::Pubid::Itu::Identifiers::Amendment, ::Pubid::Itu::Identifiers::Corrigendum,
+        ::Pubid::Itu::Identifiers::Errata, ::Pubid::Itu::Identifiers::Addendum,
+      ].freeze
+
+      # Parts of a recommendation that print its date first when it has one,
+      # "ITU-T Z.100 (06/2021) Annex F1", and their own date otherwise.
+      ANNEXED = [
+        ::Pubid::Itu::Identifiers::AnnexOfRecommendation,
+        ::Pubid::Itu::Identifiers::AppendixOfRecommendation,
+      ].freeze
+
+      # @param refid [Pubid::Itu::Identifier, String] a document reference; a
+      #   String that Pubid::Itu cannot parse raises Pubid::Errors::ParseError
       # @return [Relaton::Itu::HitCollection]
       def search(refid)
         if refid.is_a? String
           warn_incorrect_ref(refid)
-          refid = Pubid.parse refid
+          refid = ::Pubid::Itu.parse refid
         end
         HitCollection.new(refid).tap(&:search)
       end
 
-      # @param code [String] the ITU standard Code to look up
+      # @param ref [String, Pubid::Itu::Identifier] the ITU standard Code to
+      #   look up, or its parse from Relaton::Db (relaton#205); #with_year
+      #   copies, so the pubid is not changed
       # @param year [String] the year the standard was published (optional)
       # @param opts [Hash] options
       # @return [Relaton::Bib::ItemData, nil]
-      def get(code, year = nil, opts = {})
-        warn_incorrect_ref(code)
-        refid = Pubid.parse code
-        refid.year ||= year
+      def get(ref, year = nil, opts = {})
+        if ref.is_a? String
+          warn_incorrect_ref(ref)
+          ref = ::Pubid::Itu.parse(ref)
+        end
+        refid = with_year ref, year
 
         ret = itubib_get1(refid)
         return nil if ret.nil?
 
-        ret = ret.to_most_recent_reference unless refid.year || opts[:keep_year]
+        unless edition_year(refid) || opts[:keep_year]
+          ret = ret.to_most_recent_reference
+        end
         ret = ret.to_all_parts if opts[:all_parts]
         ret
       end
 
+      # The year a reference selects its hits by. It is read the way a hit
+      # code's first date is: for an amending supplement, it is the year of
+      # the recommendation it amends. HitCollection uses it too.
+      #
+      # @param refid [Pubid::Itu::Identifier]
+      # @return [String, nil]
+      def edition_year(refid)
+        return refid.base&.year if amending?(refid)
+        return refid.base&.year || refid.year if annexed?(refid)
+
+        refid.year
+      end
+
+      # The identifier a reference is a part of: the recommendation an
+      # amending supplement, an annex or an appendix belongs to; else itself.
+      # A series or recommendation supplement is a document of its own.
+      #
+      # @param refid [Pubid::Itu::Identifier]
+      # @return [Pubid::Itu::Identifier]
+      def document(refid)
+        amending?(refid) || annexed?(refid) ? refid.base : refid
+      end
+
       private
 
+      # @param refid [Pubid::Itu::Identifier]
+      # @param year [String, nil] the year passed beside the reference
+      # @return [Pubid::Itu::Identifier] refid, dated with year when it has
+      #   no year of its own
+      def with_year(refid, year)
+        return refid if year.nil? || edition_year(refid)
+
+        hash = refid.to_hash
+        dated = amending?(refid) ? hash["base"] : hash
+        dated["year"] = year.to_s
+        ::Pubid::Itu::Identifier.from_hash hash
+      end
+
+      def amending?(refid) = AMENDING.any? { |klass| refid.is_a? klass }
+
+      def annexed?(refid) = ANNEXED.any? { |klass| refid.is_a? klass }
+
       def warn_incorrect_ref(ref)
         if ref =~ /(ITU[\s-]T\s\w)\.(Suppl\.|Annex)\s?(\w?\d+)/
           correct_ref = "#{$~[1]} #{$~[2]} #{$~[3]}"
@@ -45,30 +107,59 @@ module Relaton
         Util.info "Not found.", key: refid.to_s
         # one hit per edition means a year can repeat (a recommendation and its
         # amendment); the requested year itself is not a "different year" hint
-        years = missed_years.uniq.reject { |y| y == refid.year }
+        year = edition_year(refid)
+        years = missed_years.uniq.reject { |y| y == year }
         if years.any?
           plural = years.size > 1 ? "s" : ""
-          Util.info "There was no match for `#{refid.year}` year, though there were matches " \
-                    "found for `#{years.join('`, `')}` year#{plural}.", key: refid.to_s
+          Util.info "There was no match for `#{year}` year, though there " \
+                    "were matches found for `#{years.join('`, `')}` " \
+                    "year#{plural}.", key: refid.to_s
         end
         nil
       end
 
+      # Keep the hits that are the referenced document, in any edition:
+      # #isobib_results_filter picks the year afterwards. The date is ignored,
+      # and the version only when the reference names none. Everything else
+      # must be equal, so a supplement, or a Report for a Recommendation, never
+      # matches.
+      #
+      # Not `===`: it reads a missing `subseries` as "any value", so
+      # `ITU-T H.264` would keep `ITU-T H.264.1`.
       def search_filter(refid)
-        result = search(refid)
-        result.select do |i|
+        ignore = %i[year month day]
+        ignore << :version unless stated_version?(refid)
+        search(refid).select do |i|
           next true unless i.hit[:code]
 
-          pubid = Pubid.parse i.hit[:code]
-          refid.===(pubid, [:year])
+          hit = ::Pubid::Itu.parse(i.hit[:code])
+          refid.matches?(hit, ignore: ignore) && same_own_date?(refid, hit)
+        rescue ::Pubid::Errors::ParseError
+          false # a live hit ITU spells in a form nothing can identify
         end
       end
 
+      # A version on the reference, or on the recommendation it amends.
+      def stated_version?(refid)
+        id = document(refid)
+        id.respond_to?(:version) && !id.version.nil?
+      end
+
+      # `ignore:` drops the date at every level, but an amending supplement's
+      # own date tells two "Amd. 1" to different editions apart.
+      def same_own_date?(refid, hit)
+        return true unless amending?(refid)
+
+        own = refid.to_hash.slice("year", "month", "day")
+        own.all? { |key, value| hit.to_hash[key] == value }
+      end
+
       def isobib_results_filter(result, refid)
         missed_years = []
+        year = edition_year(refid)
         result.each do |r|
           /\((?:\d{2}\/)?(?<pyear>\d{4})\)/ =~ r.hit[:code]
-          if !refid.year || refid.year == pyear
+          if !year || year == pyear
             ret = r.item
             return { ret: ret } if ret
           end

data/lib/relaton/itu/data_crawler_r.rb

@@ -564,7 +564,7 @@ module Relaton
       # the first cell, the title in the second.
       #
       # @param url [String]
-      # @return [Array<Array(String, Nokogiri::XML::Element, Nokogiri::XML::NodeSet)>]
+      # @return [Array<Array(String, Moxml::Element, Moxml::NodeSet)>]
       def rows(url)
         get(url).search("//a[contains(@href,'parent=R-')]").filter_map do |a|
           id = a[:href].to_s[PARENT_RE, :id]
@@ -576,7 +576,7 @@ module Relaton
       # The title cell carries an optional red annotation ("Note - Suppressed on
       # …"), which is not part of the title.
       #
-      # @param cell [Nokogiri::XML::Element, nil]
+      # @param cell [Moxml::Element, nil]
       # @return [String]
       def title_text(cell)
         return "" unless cell