relaton 3.0.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/doi/parser.rb

@@ -73,8 +73,14 @@ module Relaton
       ATTRS = %i[type fetched title docidentifier date source abstract contributor place
                  ext relation extent series medium].freeze
 
+      # Matches content that looks like markup. Same shape as the tag detector
+      # of `Relaton::Bib::Sanitizer`, so both agree on what to parse.
+      TAG_RE = %r{<[a-zA-Z/!?]}
+
       CROSSREF_API_URL = "https://api.crossref.org/works?query=%{query}&filter=%{filter}".freeze
       MAX_RETRIES = 3
+      RATE_LIMITED = "429".freeze
+      MAX_RETRY_DELAY = 60
 
       #
       # Initialize instance.
@@ -209,10 +215,48 @@ module Relaton
       # @return [Bib::Title] The title.
       #
       def create_title(title, type = "main")
-        cnt = str_cleanup CGI.unescapeHTML(title)
+        cnt = str_cleanup normalize_markup(title)
         Bib::Title.new type: type, content: cnt, script: "Latn"
       end
 
+      #
+      # Prepare raw Crossref markup for the relaton-bib sanitizer.
+      #
+      # Crossref returns JATS markup, and it sometimes encodes that markup as
+      # HTML entities, so decode them before the content reaches a model
+      # attribute. The markup also carries the `jats:` namespace prefix, and
+      # sometimes an `xlink:` prefix on an attribute, which Relaton never
+      # declares. Removing those prefixes is `Relaton::Bib::Sanitizer`'s job:
+      # every `Bib::Title` and `Bib::Abstract` runs the sanitizer on
+      # assignment, and it declares each undeclared prefix on a wrapper
+      # element so the markup parses and maps to the basicdoc set. An
+      # undeclared prefix that survives into the output makes
+      # relaton-render return nil, and isodoc then raises. See
+      # metanorma-pdfa#99.
+      #
+      # Upstream relaton-doi carries its own prefix removal, because the
+      # sanitizer fix is not in a released relaton-bib. Here both ship in one
+      # gem, so this method is the entity decode only.
+      #
+      # Decode only content that carries no markup of its own. Crossref
+      # escapes an ampersand and a literal angle bracket *inside* real
+      # markup, so an unconditional decode turns `<jats:p>Smith &amp;
+      # Jones</jats:p>` into markup that no longer parses. The sanitizer
+      # would then give up on it and an unescapable string would reach the
+      # output -- the metanorma-pdfa#99 failure again, on the far more
+      # common input.
+      #
+      # @param [String, nil] str The raw Crossref content.
+      #
+      # @return [String, nil] The content, with the HTML entities decoded
+      #   when it holds no markup.
+      #
+      def normalize_markup(str)
+        return str if str.nil? || str.match?(TAG_RE)
+
+        CGI.unescapeHTML(str)
+      end
+
       #
       # Parse a docidentifier from the source hash.
       #
@@ -303,9 +347,10 @@ module Relaton
       def parse_abstract
         return [] unless @src["abstract"]
 
-        content = @src["abstract"]
         abstract = Bib::Abstract.new(
-          content: content, language: "en", script: "Latn",
+          content: normalize_markup(@src["abstract"]),
+          language: "en",
+          script: "Latn",
         )
         [abstract]
       end
@@ -448,8 +493,10 @@ module Relaton
       # @return [Bib::Organization] The organization.
       #
       def create_org(name, abbreviation = nil)
-        n = [Bib::TypedLocalizedString.new(content: name)]
-        a = abbreviation ? Bib::LocalizedString.new(content: abbreviation) : nil
+        n = [Bib::TypedLocalizedString.new(content: normalize_markup(name))]
+        a = if abbreviation
+              Bib::LocalizedString.new(content: normalize_markup(abbreviation))
+            end
         Bib::Organization.new name: n, abbreviation: a
       end
 
@@ -462,11 +509,7 @@ module Relaton
         return [] unless @src["standards-body"]
 
         name, acronym = @src["standards-body"].values_at("name", "acronym")
-        org = create_org(
-          CGI.unescapeHTML(name),
-          acronym && CGI.unescapeHTML(acronym),
-        )
-        [contributor(org, "authorizer")]
+        [contributor(create_org(name, acronym), "authorizer")]
       end
 
       #
@@ -728,7 +771,8 @@ module Relaton
 
         @src["container-title"].map do |ct|
           contrib = create_authors_editors false, "editor"
-          bib = Bib::ItemBase.new(title: [Bib::Title.new(content: ct)], contributor: contrib)
+          title = Bib::Title.new content: normalize_markup(ct)
+          bib = Bib::ItemBase.new(title: [title], contributor: contrib)
           Bib::Relation.new(type: "includedIn", bibitem: bib)
         end
       end
@@ -777,7 +821,7 @@ module Relaton
                   else []
                   end
         con_ttl.map do |ct|
-          title = Bib::Title.new content: ct
+          title = Bib::Title.new content: normalize_markup(ct)
           Bib::Series.new title: [title], abbreviation: abbrev
         end
       end
@@ -800,17 +844,36 @@ module Relaton
       # @param [String] query The query string.
       # @param [String] filter The filter string.
       #
-      # @return [Array<Hash>, nil] Items array from response or nil for 4xx responses.
+      # @return [Array<Hash>, nil] Items array from response or nil for 4xx responses
+      #   other than 429.
       #
-      # @raise [Relaton::RequestError] If request fails after retries.
+      # @raise [Relaton::RequestError] If request fails after retries, or if the
+      #   rate limit does not clear.
       #
-      def fetch_crossref(query:, filter:) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
+      def fetch_crossref(query:, filter:) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
         url = format(CROSSREF_API_URL, query: query, filter: filter)
         retries = 0
+        throttled = 0
         begin
           resp = Crossref.agent.get url
           JSON.parse(resp.body).dig("message", "items")
         rescue Mechanize::ResponseCodeError => e
+          # 429 must be handled before the 4xx nil return: a throttle is not
+          # "no such record". Returning nil here made a rate-limited lookup look
+          # like a missing document, so callers such as #parent_item silently
+          # produced an incomplete item instead of failing.
+          if e.response_code == RATE_LIMITED
+            throttled += 1
+            if throttled <= MAX_RETRIES
+              # Cap the backoff itself, not just the header value: escalating by
+              # attempt would otherwise let a single `Retry-After: 60` stall one
+              # lookup for 60+120+180s.
+              sleep [retry_delay(e) * throttled, MAX_RETRY_DELAY].min
+              retry
+            end
+            raise Relaton::RequestError,
+                  "Crossref rate limit not cleared after #{MAX_RETRIES} retries: #{url}"
+          end
           return nil if e.response_code.start_with?("4")
 
           raise Relaton::RequestError, "Crossref request failed: #{e.response_code} #{e.page.body}"
@@ -822,6 +885,37 @@ module Relaton
           raise Relaton::RequestError, "Crossref JSON parsing error: #{e.message}"
         end
       end
+
+      #
+      # Base seconds to wait before retrying a throttled Crossref request.
+      # Prefers Retry-After, falls back to Crossref's X-Rate-Limit-Interval
+      # (formatted "1s", hence the digit scan). An empty-bodied 429 carries
+      # neither header, so this never assumes one is present.
+      #
+      # @param [Mechanize::ResponseCodeError] err The rate-limit error.
+      #
+      # @return [Integer] Delay in seconds, at least 1 and at most MAX_RETRY_DELAY.
+      #
+      def retry_delay(err)
+        headers = err.page&.response || {}
+        secs = retry_after_seconds(headers["retry-after"]) ||
+               headers["x-rate-limit-interval"].to_s[/\d+/].to_i
+        secs.positive? ? [secs, MAX_RETRY_DELAY].min : 1
+      end
+
+      #
+      # Parse a Retry-After header value, which RFC 9110 allows to be either
+      # delta-seconds or an HTTP-date. Only the numeric form is usable as a
+      # delay: scanning digits out of a date would pick up the day-of-month,
+      # silently turning a long server-mandated backoff into a few seconds.
+      #
+      # @param [String, nil] value The raw header value.
+      #
+      # @return [Integer, nil] Seconds, or nil if absent or in the date form.
+      #
+      def retry_after_seconds(value)
+        value.to_s.strip.match?(/\A\d+\z/) ? value.to_i : nil
+      end
     end
   end
 end

data/lib/relaton/easc/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Easc
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/easc/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Easc
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/easc/bibliography.rb

@@ -0,0 +1,95 @@
+require "net/http"
+
+module Relaton
+  module Easc
+    # Index-backed retrieval for the EASC flavor (no scraping). Mirrors
+    # Relaton::Oiml::Bibliography: it parses the reference with
+    # Pubid::Easc, narrows the pre-built `relaton-data-easc` index by
+    # number, fetches the matching per-document YAML over Net::HTTP, and
+    # deserializes it into a Relaton::Easc::Item.
+    module Bibliography
+      ENDPOINT = "https://raw.githubusercontent.com/relaton/relaton-data-easc/main/".freeze
+
+      class << self
+        #
+        # Search for an EASC publication by its identifier.
+        #
+        # @param text [String, Pubid::Easc::Identifier] the EASC reference to
+        #   look up (e.g. "ПМГ 03-2025", "РМГ 151-2025", or the Latin
+        #   transliteration "PMG 03-2025")
+        # @param year [String, nil] the edition year (optional; usually embedded
+        #   in the reference)
+        # @param _opts [Hash] options (unused)
+        #
+        # @return [Relaton::Easc::Item, nil] the publication or nil if not found
+        #
+        def search(text, year = nil, _opts = {})
+          pubid = text.is_a?(String) ? ::Pubid::Easc.parse(text) : text
+          Util.info "Fetching from Relaton repository ...", key: pubid.to_s
+          # Pass the pubid so Relaton::Index narrows candidates by number via
+          # binary search before applying the block. Every row's `:id` is a
+          # Pubid::Easc::Identifier (deserialized via the `pubid_class` passed to
+          # #index), so the block compares pubids and picks the latest edition.
+          row = index.search(pubid) { |r| pubid_match?(r[:id], pubid, year) }
+                     .max_by { |r| r[:id].year.to_i }
+          unless row
+            Util.info "Not found.", key: pubid.to_s
+            return
+          end
+
+          uri = URI("#{ENDPOINT}#{row[:file]}")
+          resp = Net::HTTP.get_response uri
+          unless resp.code == "200"
+            raise Relaton::RequestError, "Could not access #{uri}: HTTP #{resp.code}"
+          end
+
+          item = Relaton::Easc::Item.from_yaml resp.body
+          Util.info "Found: `#{item.docidentifier.first&.content}`", key: pubid.to_s
+          item.tap { |i| i.fetched = Date.today.to_s }
+        rescue SocketError, Errno::EINVAL, Errno::ECONNRESET, EOFError,
+               Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
+               Net::ProtocolError, Net::ReadTimeout, OpenSSL::SSL::SSLError,
+               Errno::ETIMEDOUT => e
+          raise Relaton::RequestError, "Could not access #{uri}: #{e.message}"
+        end
+
+        #
+        # Fetch an EASC publication. EASC citations are always dated
+        # (e.g. "ПМГ 03-2025"), so there is no undated/most-recent handling to
+        # mirror from the ISO/OIML fetchers — `get` just returns what `search`
+        # resolves.
+        #
+        # @see #search
+        #
+        def get(ref, year = nil, opts = {})
+          search(ref, year, opts)
+        end
+
+        private
+
+        def index
+          Relaton::Index.find_or_create(
+            :easc,
+            url: "#{ENDPOINT}#{INDEXFILE}.zip",
+            file: "#{INDEXFILE}.yaml",
+            pubid_class: ::Pubid::Easc::Identifier,
+          )
+        end
+
+        # Both `row_id` and `query` are Pubid::Easc::Identifier instances. A row
+        # matches when its series, defense variant, and number are equal (so a
+        # ПМГ never matches a РМГ, and the «В» defense variant never matches the
+        # plain series). Year is nil-tolerant: an unqualified query finds the
+        # latest edition (selected by `max_by` in #search); the `year` argument
+        # lets a caller pin an edition the reference string omitted.
+        def pubid_match?(row_id, query, year)
+          wanted_year = (query.year || year)&.to_s
+          row_id.series == query.series &&
+            row_id.variant.to_s == query.variant.to_s &&
+            row_id.number.to_s == query.number.to_s &&
+            (wanted_year.nil? || row_id.year.to_s == wanted_year)
+        end
+      end
+    end
+  end
+end

data/lib/relaton/easc/docidentifier.rb

@@ -0,0 +1,100 @@
+module Relaton
+  module Easc
+    # Structured EASC document identifier. The docid string is the canonical
+    # citation form (e.g. "ПМГ 03-2025", "РМГ 151-2025"), parsed into a
+    # `Pubid::Easc::Identifier` kept in `@pubid` while the lutaml `content`
+    # attribute stays a plain string for serialization.
+    #
+    # Parsing is **soft**: `content=` lazily requires pubid and rescues
+    # `LoadError`/`StandardError`, so a missing gem or non-EASC content leaves
+    # `@pubid` nil rather than raising.
+    #
+    # This was an empty subclass, which meant it inherited
+    # `Bib::Docidentifier`'s abstract `remove_part!` / `remove_date!` /
+    # `to_all_parts!`. `Bib::ItemData` broadcasts all three to every
+    # docidentifier, so `#to_all_parts` and `#to_most_recent_reference` raised
+    # `NotImplementedError` on **every** EASC item — and it descends from
+    # `ScriptError`, so a caller's `rescue => e` did not catch it.
+    class Docidentifier < Bib::Docidentifier
+      attr_reader :pubid
+
+      # Capture the inherited content setter before overriding #content=, so
+      # #refresh_content! can write the re-rendered string back WITHOUT
+      # re-parsing (a re-parse would rebuild @pubid and discard in-place
+      # mutations such as `all_parts = true`).
+      alias_method :store_content, :content=
+
+      def initialize(attrs = {}, options = {})
+        pubid = attrs.is_a?(Hash) ? attrs.delete(:pubid) : nil
+        attrs[:content] ||= pubid.to_s if pubid
+        super
+        @pubid = pubid if pubid
+      end
+
+      def content=(value)
+        super
+        return unless value
+
+        @pubid = begin
+          require "pubid"
+          ::Pubid::Easc.parse(value)
+        rescue LoadError, StandardError
+          nil
+        end
+      end
+
+      # `Pubid::Easc::Identifier` models publisher/series/variant/number/year,
+      # so the mapping is EASC-specific:
+      #
+      # - **`remove_date!` → clears `year`.** EASC carries no separate date
+      #   component; the year is the edition discriminator, so clearing it
+      #   yields the undated ("most recent") reference: `ПМГ 03-2025` →
+      #   `ПМГ 03`.
+      # - **`remove_part!` → clears the (unused) `part`/`subpart`.** EASC models
+      #   no part, so this is a no-op for the rendered string; implemented so it
+      #   never raises, and it starts working if pubid-easc ever adds one.
+      # - **`to_all_parts!` → both, then wraps `@pubid` in pubid's `AllParts`.**
+      #   `content` stays the plain stripped id — EASC's own renderer has no
+      #   "(all parts)" marker, and `content` is cached from the pre-wrap
+      #   pubid on purpose. `#pubid` itself, read directly, now answers
+      #   `all_parts? == true` and renders WITH pubid's generic marker
+      #   (`#to_s`), since it's the wrapper.
+      #
+      # All three no-op safely when `@pubid` is nil.
+
+      def remove_part!
+        clear_attr!(:part)
+      end
+
+      def remove_date!
+        clear_attr!(:year)
+      end
+
+      def to_all_parts!
+        return if !@pubid || @pubid.all_parts?
+
+        # `#exclude` (no args) rebuilds a full independent copy — `remove_part!`
+        # / `remove_date!` mutate `@pubid` in place, so a bare reference here
+        # would lose the original year to that mutation too.
+        original = @pubid.exclude
+        remove_part!
+        remove_date!
+        @pubid = original.to_all_parts
+      end
+
+      private
+
+      def clear_attr!(attr)
+        return unless @pubid && @pubid.respond_to?("#{attr}=")
+
+        @pubid.public_send("#{attr}=", nil)
+        @pubid.subpart = nil if attr == :part && @pubid.respond_to?(:subpart=)
+        refresh_content!
+      end
+
+      def refresh_content!
+        store_content(@pubid.to_s) if @pubid
+      end
+    end
+  end
+end

data/lib/relaton/easc/doctype.rb

@@ -0,0 +1,14 @@
+module Relaton
+  module Easc
+    # EASC document types. Two series, both issued by the Eurasian
+    # Economic Standards Council:
+    #   * PMG — ПМГ (Правила по межгосударственной стандартизации)
+    #   * RMG — РМГ (Рекомендации по межгосударственной стандартизации)
+    class Doctype < Bib::Doctype
+      TYPES = %w[
+        pmg
+        rmg
+      ].freeze
+    end
+  end
+end

data/lib/relaton/easc/ext.rb

@@ -0,0 +1,44 @@
+module Relaton
+  module Easc
+    # EASC bibliographic item extension. Adds EASC-specific structured
+    # metadata on top of the shared Relaton::Bib::Ext so the fields
+    # round-trip through both XML and YAML without per-repo merge hacks.
+    #
+    # Fields:
+    #   * urn             — urn:easc:<series>:...
+    #   * webpage         — mgscatalog.by detail URL
+    #   * session         — МГС council session that adopted the doc
+    #                       (e.g. "67МГС", "19МГС")
+    #   * developer       — Разработчик (e.g. "Российская Федерация")
+    #   * joining_states  — country codes of states that joined
+    #                       (collection: e.g. ["АРМ", "БЕИ", "КАЗ"])
+    #   * assigned_to     — Закреплен за (e.g. "МТК 536 Методология...")
+    class Ext < Bib::Ext
+      attribute :doctype,        Doctype
+      attribute :urn,            :string
+      attribute :webpage,        :string
+      attribute :session,        :string
+      attribute :developer,      :string
+      attribute :joining_states, :string, collection: true, initialize_empty: true
+      attribute :assigned_to,    :string
+
+      xml do
+        map_element "urn",             to: :urn
+        map_element "webpage",         to: :webpage
+        map_element "session",         to: :session
+        map_element "developer",       to: :developer
+        map_element "joining_states",  to: :joining_states
+        map_element "assigned_to",     to: :assigned_to
+      end
+
+      key_value do
+        map_element "urn",             to: :urn
+        map_element "webpage",         to: :webpage
+        map_element "session",         to: :session
+        map_element "developer",       to: :developer
+        map_element "joining_states",  to: :joining_states
+        map_element "assigned_to",     to: :assigned_to
+      end
+    end
+  end
+end

data/lib/relaton/easc/item.rb

@@ -0,0 +1,13 @@
+module Relaton
+  module Easc
+    # EASC bibliographic item. Re-declares +ext+ to use the typed Ext
+    # subclass so EASC-specific fields round-trip natively.
+    class Item < Bib::Item
+      model ItemData
+
+      attribute :ext, Ext
+      attribute :docidentifier, Docidentifier, collection: true,
+                                               initialize_empty: true
+    end
+  end
+end

data/lib/relaton/easc/item_base.rb

@@ -0,0 +1,18 @@
+module Relaton
+  module Easc
+    class ItemBase < Lutaml::Model::Serializable
+      include Bib::NamespaceHelper
+
+      attr_accessor :type
+
+      model ItemData
+
+      instance_exec(&Bib::ItemShared::ATTRIBUTES)
+
+      xml do
+        map_attribute "type", to: :type
+        instance_exec(&Bib::ItemShared::XML_BODY)
+      end
+    end
+  end
+end

data/lib/relaton/easc/item_data.rb

@@ -0,0 +1,6 @@
+module Relaton
+  module Easc
+    class ItemData < Bib::ItemData
+    end
+  end
+end

data/lib/relaton/easc/processor.rb

@@ -0,0 +1,46 @@
+require "relaton/core/processor"
+
+module Relaton
+  module Easc
+    # Relaton processor for the EASC flavor. Registered by
+    # Relaton::Db::Registry when the unified gem loads (see
+    # lib/relaton/db/registry.rb). Supports `relaton fetch easc ...`
+    # once the relaton-data-easc dataset is installed.
+    class Processor < Core::Processor
+      attr_reader :idtype
+
+      def initialize
+        @short = :relaton_easc
+        @prefix = "EASC"
+        # Both Cyrillic and Latin series prefixes route here.
+        @defaultprefix = %r{^(?:ПМГ|РМГ|PMG|RMG)\b}
+        @idtype = "EASC"
+      end
+
+      def get(code, date, opts)
+        require_relative "../easc"
+        Bibliography.get(code, date, opts)
+      end
+
+      def from_xml(xml)
+        require_relative "../easc"
+        Item.from_xml xml
+      end
+
+      def from_yaml(yaml)
+        require_relative "../easc"
+        Item.from_yaml(yaml)
+      end
+
+      def grammar_hash
+        require_relative "../easc"
+        @grammar_hash ||= ::Relaton::Easc.grammar_hash
+      end
+
+      def remove_index_file
+        require_relative "../easc"
+        Relaton::Index.find_or_create(:easc, url: true, file: "#{INDEXFILE}.yaml").remove_file
+      end
+    end
+  end
+end

data/lib/relaton/easc/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Easc
+    module Util
+      extend Relaton::Bib::Util
+      PROGNAME = "relaton-easc".freeze
+    end
+  end
+end

data/lib/relaton/easc.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "relaton/bib"
+require "relaton/index"
+require "relaton/version"
+require "pubid"
+
+# Relaton::Easc is the EASC (Евразийский экономический совет /
+# Eurasian Economic Standards Council) flavor of the unified relaton
+# gem. Adds typed Ext fields (urn, webpage, session, developer,
+# joining_states) so they round-trip natively through YAML and XML
+# without per-repo merge hacks.
+module Relaton
+  module Easc
+    INDEXFILE = "index-v2".freeze
+
+    class Error < StandardError; end
+
+    autoload :Doctype,       "relaton/easc/doctype"
+    autoload :Ext,           "relaton/easc/ext"
+    autoload :ItemData,      "relaton/easc/item_data"
+    autoload :ItemBase,      "relaton/easc/item_base"
+    autoload :Item,          "relaton/easc/item"
+    autoload :Bibitem,       "relaton/easc/bibitem"
+    autoload :Bibdata,       "relaton/easc/bibdata"
+    autoload :Docidentifier, "relaton/easc/docidentifier"
+    autoload :Util,          "relaton/easc/util"
+    autoload :Bibliography,  "relaton/easc/bibliography"
+    autoload :Processor,     "relaton/easc/processor"
+
+    def self.grammar_hash
+      Digest::MD5.hexdigest Relaton::VERSION
+    end
+  end
+end