relaton-oasis 2.1.1 → 2.2.0.pre.alpha.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c3434e03a8bfad2eb1ae188cbc3e27908a530661267335656356e530889a3f21
-  data.tar.gz: 2a81653e2ce01d964ec217b0d2ec0d1fd18bcc38fa4065f1ffb1029340d9a615
+  metadata.gz: ac8025bbbca70581bc6047bcbef3b1d81b9f110b5d91a235d43dd00201052864
+  data.tar.gz: 04db6b5abd8c83acfcdb46916880a47e210a98bf2c78d24c1e9809e3c7a2fd0f
 SHA512:
-  metadata.gz: 43cec70fb75e28fcfaa9d5203c95cc2536a9050a952427192c8cf8d133b86742a98aac9b73e11884133b19f035134249af407fe91ec820b1b0aa1769121e9699
-  data.tar.gz: 8256710e4b5699c8cefec19f0cf8919dc4d703c05d461c71a896a7ff0e1497923d7dfc0b78db59b882e07b983bcff8799aafae830b93da4c93ff0240f699e93f
+  metadata.gz: cee78c32f3033e7d6fd69c0697598225147b6e58329dce7fd9015b323f3b5aaec201080cb5427d80e1ab86126b48714a0f4ac78d2ff2e9bfe151af13117cc1ff
+  data.tar.gz: cc409df22b3c451c748d84cfe4cdf4b41f67ba338071b147e5d0b5766101306cc86b95f2ac9add1c6e3d00ce64eaa06779998de897f5cdf7aa1d4e8bedb8facc

data/Gemfile

@@ -5,6 +5,14 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in relaton_oasis.gemspec
 gemspec
 
+# Use local monorepo sibling gems where available.
+Dir["../*/"].each do |dir|
+  name = File.basename(dir)
+  next if name == File.basename(__dir__)
+  next unless File.exist?(File.join(dir, "#{name}.gemspec"))
+  gem name, path: dir
+end
+
 
 gem "rake", "~> 13.0"
 gem "rspec", "~> 3.0"

data/lib/relaton/oasis/browser_agent.rb

@@ -0,0 +1,71 @@
+require "ferrum"
+require "nokogiri"
+
+module Relaton
+  module Oasis
+    # Thin Ferrum-backed agent that drives headless Chrome with stealth tweaks
+    # so the Cloudflare-protected oasis-open.org host serves real HTML instead
+    # of a "Just a moment..." challenge. Mirrors the pattern used by
+    # Relaton::Cie::BrowserAgent.
+    class BrowserAgent
+      UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " \
+           "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
+      CHALLENGE_MARKERS = ["Just a moment", "challenge-platform"].freeze
+      MAX_CHALLENGE_WAIT = 30
+
+      def initialize
+        @browser = Ferrum::Browser.new(
+          headless: true,
+          timeout: 90,
+          process_timeout: 90,
+          window_size: [1366, 768],
+          browser_options: {
+            "disable-blink-features" => "AutomationControlled",
+            "disable-quic" => nil,
+            "no-sandbox" => nil,
+          },
+        )
+        @browser.headers.set(
+          "Accept-Language" => "en-US,en;q=0.9",
+          "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9," \
+                      "image/webp,*/*;q=0.8",
+          "User-Agent" => UA,
+        )
+        @browser.evaluate_on_new_document(<<~JS)
+          Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
+          Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
+          Object.defineProperty(navigator, 'plugins', { get: () => [1,2,3,4,5] });
+          window.chrome = { runtime: {} };
+        JS
+      end
+
+      def get(url)
+        @browser.go_to(url)
+        wait_for_challenge
+        Nokogiri::HTML(@browser.body)
+      end
+
+      # HTTP status code of the most recent navigation's main resource,
+      # or nil if no navigation has happened yet.
+      def last_status
+        @browser&.network&.status
+      end
+
+      def quit
+        @browser&.quit
+      ensure
+        @browser = nil
+      end
+
+      private
+
+      def wait_for_challenge
+        MAX_CHALLENGE_WAIT.times do
+          return unless CHALLENGE_MARKERS.any? { |m| @browser.body.include?(m) }
+
+          sleep 1
+        end
+      end
+    end
+  end
+end

data/lib/relaton/oasis/data_fetcher.rb

@@ -1,5 +1,5 @@
-require "mechanize"
 require_relative "../oasis"
+require_relative "browser_agent"
 require_relative "data_parser_utils"
 require_relative "data_parser"
 require_relative "data_part_parser"
@@ -7,24 +7,47 @@ require_relative "data_part_parser"
 module Relaton
   module Oasis
     class DataFetcher < Core::DataFetcher
+      STANDARDS_URL = "https://www.oasis-open.org/standards/".freeze
+      RETRIABLE_ERRORS = [
+        SocketError,
+        Ferrum::TimeoutError,
+        Ferrum::PendingConnectionsError,
+        Ferrum::StatusError,
+      ].freeze
+
       def log_error(msg)
         Util.error msg
       end
 
       def fetch(_source = nil)
-        agent = Mechanize.new
-        resp = agent.get "https://www.oasis-open.org/standards/"
-        doc = Nokogiri::HTML resp.body
+        doc = with_retry { agent.get(STANDARDS_URL) }
         doc.xpath("//details").map do |item|
-          save_doc DataParser.new(item, @errors).parse
+          save_doc DataParser.new(item, @errors, agent: agent).parse
           fetch_parts item
         end
         index.save
         report_errors
+      ensure
+        @agent&.quit
       end
 
       private
 
+      def agent
+        @agent ||= BrowserAgent.new
+      end
+
+      def with_retry
+        tries = 0
+        begin
+          tries += 1
+          yield
+        rescue *RETRIABLE_ERRORS => e
+          retry if tries < 4
+          raise e
+        end
+      end
+
       def index
         @index ||= Relaton::Index.find_or_create(
           :oasis, file: "#{INDEXFILE}.yaml"
@@ -38,7 +61,9 @@ module Relaton
         parts = item.xpath(xpath)
         return unless parts.size > 1
 
-        parts.each { |part| save_doc DataPartParser.new(part, @errors).parse }
+        parts.each do |part|
+          save_doc DataPartParser.new(part, @errors, agent: agent).parse
+        end
       end
 
       def save_doc(doc) # rubocop:disable Metrics/AbcSize

data/lib/relaton/oasis/data_parser.rb

@@ -9,9 +9,10 @@ module Relaton
       #
       # @param [Nokogiri::HTML::Element] node document node
       #
-      def initialize(node, errors = {})
+      def initialize(node, errors = {}, agent: nil)
         @node = node
         @errors = errors
+        @agent = agent
       end
 
       def title
@@ -158,7 +159,7 @@ module Relaton
         rels = @node.xpath(xpath)
         result = if rels.size > 1
                    rels.map do |r|
-                     docid = DataPartParser.new(r).parse_docid
+                     docid = DataPartParser.new(r, @errors, agent: @agent).parse_docid
                      bib = ItemData.new(formattedref: Bib::Formattedref.new(content: docid[0].content))
                      Bib::Relation.new(type: "hasPart", bibitem: bib)
                    end

data/lib/relaton/oasis/data_parser_utils.rb

@@ -1,7 +1,17 @@
+require "mechanize"
+require_relative "browser_agent"
+
 module Relaton
   module Oasis
     # Common methods for document and part parsers.
     module DataParserUtils
+      RETRIABLE_PAGE_ERRORS = [
+        Errno::ETIMEDOUT,
+        Net::OpenTimeout,
+        Ferrum::TimeoutError,
+        Ferrum::PendingConnectionsError,
+        Ferrum::StatusError,
+      ].freeze
       #
       # Parse contributor.
       #
@@ -50,9 +60,17 @@ module Relaton
       def page
         return @page if defined? @page
 
-        if link_node && link_node[:href].match?(/\.html$/)
+        @page = nil
+        return @page unless link_node && link_node[:href].match?(/\.html$/)
+
+        if @agent
+          doc = retry_page(link_node[:href], @agent)
+          @page = doc if doc && @agent.last_status == 200
+        else
+          # No injected agent (e.g. unit tests with VCR cassettes): fall back
+          # to a Mechanize request — VCR can intercept it.
           agent = Mechanize.new
-          agent.agent.allowed_error_codes = [404]
+          agent.agent.allowed_error_codes = [403, 404, 503]
           resp = retry_page(link_node[:href], agent)
           @page = resp if resp && resp.code == "200"
         end
@@ -62,15 +80,15 @@ module Relaton
       # Retry to get page.
       #
       # @param [String] url page URL
-      # @param [Mechanize] agent HTTP client
+      # @param [#get] agent HTTP client responding to #get(url)
       # @param [Integer] retries number of retries
       #
-      # @return [Mechanize::Page, nil] page or nil
+      # @return [Nokogiri::HTML::Document, Mechanize::Page, nil] page or nil
       #
       def retry_page(url, agent, retries = 3)
         sleep 1 # to avoid 429 error
         agent.get url
-      rescue Errno::ETIMEDOUT, Net::OpenTimeout => e
+      rescue *RETRIABLE_PAGE_ERRORS => e
         retry if (retries -= 1).positive?
         Util.error "Failed to get page `#{url}`\n#{e.message}"
         nil
@@ -84,7 +102,7 @@ module Relaton
                            "[starts-with(., 'Editor')]]"
                    page.xpath(xpath).map do |p|
                      create_contribution_info(p, "editor", ["Chair"])
-                   end
+                   end.compact
                  else
                    []
                  end
@@ -101,7 +119,7 @@ module Relaton
                            "[contains(@class, 'Title')]]"
                    page.xpath(xpath).map do |p|
                      create_contribution_info(p, "editor")
-                   end
+                   end.compact
                  else
                    parse_editors_from_text
                  end
@@ -111,6 +129,9 @@ module Relaton
 
       def create_contribution_info(person_node, type, description = [])
         name = person_node.text.match(/^[^(]+/).to_s.strip
+        return nil if name.empty? || !name.match?(/\A\p{L}/) ||
+                      name.match?(%r{\A(?:https?://|urn:)})
+
         email, org = person_node.xpath ".//a[@href]"
         entity = create_person name, email, org
         desc = description.map { |d| Bib::LocalizedMarkedUpString.new(content: d) }
@@ -137,7 +158,15 @@ module Relaton
           [href.split(":")[1]]
         elsif (cf_email = email.at(".//span[@data-cfemail]"))
           decoded = decode_cf_email(cf_email["data-cfemail"])
-          decoded.empty? ? [] : [decoded]
+          return [] if decoded.empty?
+
+          # Cloudflare obfuscates ASCII email characters in the data-cfemail
+          # span but leaves non-ASCII characters (e.g. the Latin "fl" ligature
+          # U+FB02) as plain text outside the span. Concatenate any sibling
+          # text and NFKC-normalize so ligatures become their ASCII equivalent.
+          prefix = cf_email.xpath("./preceding-sibling::node()").map(&:text).join
+          suffix = cf_email.xpath("./following-sibling::node()").map(&:text).join
+          [(prefix + decoded + suffix).unicode_normalize(:nfkc)]
         else
           []
         end

data/lib/relaton/oasis/data_part_parser.rb

@@ -9,9 +9,10 @@ module Relaton
       #
       # @param [Nokogiri::HTML::Element] node document node
       #
-      def initialize(node, errors = {})
+      def initialize(node, errors = {}, agent: nil)
         @node = node
         @errors = errors
+        @agent = agent
       end
 
       def text
@@ -34,11 +35,11 @@ module Relaton
         xpath = "./span[@class='citationTitle' " \
                 "or @class='citeTitle']|./em|./i"
         t = @node.at(xpath)
-        @title = if t then t.text
+        @title = if t
+                   t.text
                  else
-                   text.match(
-                     /(?<content>.+)\s(?:Edited|\d{2}\s\w+\d{4})/,
-                   )[:content]
+                   m = text.match(/(?<content>.+)\s(?:Edited|\d{2}\s\w+\d{4})/)
+                   m ? m[:content] : text
                  end.strip
       end
 
@@ -82,7 +83,7 @@ module Relaton
       # @return [String] document number
       #
       def parse_docnumber
-        ref = @node.at("./span[@class='citationLabel']/strong|./strong|b/span")
+        ref = @node.at("./span/strong|./strong|./b/span")
         num = ref.text.match(/[^\[\]]+/).to_s
         id = parse_errata(num)
         # some part refs need "Pt" to distinguish from root doc
@@ -110,8 +111,8 @@ module Relaton
       # @return [Array<Bib::Date>] bibliographic dates
       #
       def parse_date
-        /(?<on>\d{1,2}\s\w+\s\d{4})/ =~ text
-        result = [Bib::Date.new(at: Date.parse(on).to_s, type: "issued")]
+        match = text.match(/(?<on>\d{1,2}\s\w+\s\d{4})/)
+        result = match ? [Bib::Date.new(at: Date.parse(match[:on]).to_s, type: "issued")] : []
         @errors[:part_date] &&= result.empty?
         result
       end
@@ -182,7 +183,7 @@ module Relaton
       # @return [Array<Bib::Relation>] document relations
       #
       def parse_relation
-        parser = DataParser.new @node.at("./ancestor::details")
+        parser = DataParser.new(@node.at("./ancestor::details"), @errors, agent: @agent)
         fref = parser.parse_docid[0].content
         bib = ItemData.new(formattedref: Bib::Formattedref.new(content: fref))
         result = [Bib::Relation.new(type: "partOf", bibitem: bib)]

data/lib/relaton/oasis/version.rb

@@ -2,6 +2,6 @@
 
 module Relaton
   module Oasis
-    VERSION = "2.1.1"
+    VERSION = "2.2.0.pre.alpha.1"
   end
 end

data/relaton-oasis.gemspec

@@ -14,7 +14,7 @@ Gem::Specification.new do |spec|
                        "bibliographic use using the BibliographicItem model"
   spec.homepage      = "https://github.com/metanorma/relaton-oasis"
   spec.license       = "BSD-2-Clause"
-  spec.required_ruby_version = Gem::Requirement.new(">= 3.2.0")
+  spec.required_ruby_version = Gem::Requirement.new(">= 3.3.0")
 
   # spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
 
@@ -31,11 +31,11 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
+  spec.add_dependency "ferrum", "~> 0.17"
   spec.add_dependency "mechanize", "~> 2.10"
-  spec.add_dependency "multi_json", "~> 1.15.0"
-  spec.add_dependency "relaton-bib", "~> 2.1.0"
-  spec.add_dependency "relaton-core", "~> 0.0.13"
-  spec.add_dependency "relaton-index", "~> 0.2.0"
+  spec.add_dependency "relaton-bib", "~> 2.2.0.pre.alpha.1"
+  spec.add_dependency "relaton-core", "~> 2.2.0.pre.alpha.1"
+  spec.add_dependency "relaton-index", "~> 2.2.0.pre.alpha.1"
 
   # For more information and examples about making a new gem, checkout our
   # guide at: https://bundler.io/guides/creating_gem.html

metadata

@@ -1,85 +1,85 @@
 --- !ruby/object:Gem::Specification
 name: relaton-oasis
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 2.2.0.pre.alpha.1
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-05-12 00:00:00.000000000 Z
+date: 2026-06-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: mechanize
+  name: ferrum
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.10'
+        version: '0.17'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.10'
+        version: '0.17'
 - !ruby/object:Gem::Dependency
-  name: multi_json
+  name: mechanize
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.15.0
+        version: '2.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.15.0
+        version: '2.10'
 - !ruby/object:Gem::Dependency
   name: relaton-bib
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: 2.2.0.pre.alpha.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: 2.2.0.pre.alpha.1
 - !ruby/object:Gem::Dependency
   name: relaton-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.13
+        version: 2.2.0.pre.alpha.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.13
+        version: 2.2.0.pre.alpha.1
 - !ruby/object:Gem::Dependency
   name: relaton-index
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 2.2.0.pre.alpha.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 2.2.0.pre.alpha.1
 description: 'Relaton::Oasis: retrieve OASIS Standards for bibliographic use using
   the BibliographicItem model'
 email:
@@ -92,7 +92,6 @@ files:
 - ".github/workflows/release.yml"
 - ".gitignore"
 - ".rspec"
-- ".rubocop.yml"
 - CLAUDE.md
 - Gemfile
 - LICENSE.txt
@@ -101,15 +100,11 @@ files:
 - bin/console
 - bin/rspec
 - bin/setup
-- grammars/basicdoc.rng
-- grammars/biblio-standoc.rng
-- grammars/biblio.rng
-- grammars/relaton-oasis-compile.rng
-- grammars/relaton-oasis.rng
 - lib/relaton/oasis.rb
 - lib/relaton/oasis/bibdata.rb
 - lib/relaton/oasis/bibitem.rb
 - lib/relaton/oasis/bibliography.rb
+- lib/relaton/oasis/browser_agent.rb
 - lib/relaton/oasis/data_fetcher.rb
 - lib/relaton/oasis/data_parser.rb
 - lib/relaton/oasis/data_parser_utils.rb
@@ -136,7 +131,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.2.0
+      version: 3.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="

data/.rubocop.yml

@@ -1,12 +0,0 @@
-# This project follows the Ribose OSS style guide.
-# https://github.com/riboseinc/oss-guides
-# All project-specific additions and overrides should be specified in this file.
-
-require: rubocop-rails
-
-inherit_from:
-  - https://raw.githubusercontent.com/riboseinc/oss-guides/master/ci/rubocop.yml
-AllCops:
-  TargetRubyVersion: 3.2
-Rails:
-  Enabled: false