relaton-oasis 2.1.1 → 2.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c3434e03a8bfad2eb1ae188cbc3e27908a530661267335656356e530889a3f21
-  data.tar.gz: 2a81653e2ce01d964ec217b0d2ec0d1fd18bcc38fa4065f1ffb1029340d9a615
+  metadata.gz: b09904cb15aa82500946dfe0d7c3a3d3edd08c391244ef56208e077c8b322d9e
+  data.tar.gz: f16f7b98dcde027268ea89622f595c57290f6dc9845f075d240a50b9ff7996d6
 SHA512:
-  metadata.gz: 43cec70fb75e28fcfaa9d5203c95cc2536a9050a952427192c8cf8d133b86742a98aac9b73e11884133b19f035134249af407fe91ec820b1b0aa1769121e9699
-  data.tar.gz: 8256710e4b5699c8cefec19f0cf8919dc4d703c05d461c71a896a7ff0e1497923d7dfc0b78db59b882e07b983bcff8799aafae830b93da4c93ff0240f699e93f
+  metadata.gz: edbc0cb1b32818941d92dd5c5d01854205374fd4dfd23b84457bc03de3dacbd5f87c9a831a0bcf7f07ff534bae5d6a32e652fb343f48508f426ce2d1749384a7
+  data.tar.gz: 3abbf12c779c34649b0974e331ed529bd7ff65a16d2f9f7f566d96067137920c99f08c5779fcd2bfd8aecd2f07b7e7969fe4f3e7237e17ff62d29783189a6c17

data/lib/relaton/oasis/browser_agent.rb

@@ -0,0 +1,71 @@
+require "ferrum"
+require "nokogiri"
+
+module Relaton
+  module Oasis
+    # Thin Ferrum-backed agent that drives headless Chrome with stealth tweaks
+    # so the Cloudflare-protected oasis-open.org host serves real HTML instead
+    # of a "Just a moment..." challenge. Mirrors the pattern used by
+    # Relaton::Cie::BrowserAgent.
+    class BrowserAgent
+      UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " \
+           "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
+      CHALLENGE_MARKERS = ["Just a moment", "challenge-platform"].freeze
+      MAX_CHALLENGE_WAIT = 30
+
+      def initialize
+        @browser = Ferrum::Browser.new(
+          headless: true,
+          timeout: 90,
+          process_timeout: 90,
+          window_size: [1366, 768],
+          browser_options: {
+            "disable-blink-features" => "AutomationControlled",
+            "disable-quic" => nil,
+            "no-sandbox" => nil,
+          },
+        )
+        @browser.headers.set(
+          "Accept-Language" => "en-US,en;q=0.9",
+          "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9," \
+                      "image/webp,*/*;q=0.8",
+          "User-Agent" => UA,
+        )
+        @browser.evaluate_on_new_document(<<~JS)
+          Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
+          Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
+          Object.defineProperty(navigator, 'plugins', { get: () => [1,2,3,4,5] });
+          window.chrome = { runtime: {} };
+        JS
+      end
+
+      def get(url)
+        @browser.go_to(url)
+        wait_for_challenge
+        Nokogiri::HTML(@browser.body)
+      end
+
+      # HTTP status code of the most recent navigation's main resource,
+      # or nil if no navigation has happened yet.
+      def last_status
+        @browser&.network&.status
+      end
+
+      def quit
+        @browser&.quit
+      ensure
+        @browser = nil
+      end
+
+      private
+
+      def wait_for_challenge
+        MAX_CHALLENGE_WAIT.times do
+          return unless CHALLENGE_MARKERS.any? { |m| @browser.body.include?(m) }
+
+          sleep 1
+        end
+      end
+    end
+  end
+end

data/lib/relaton/oasis/data_fetcher.rb

@@ -1,5 +1,5 @@
-require "mechanize"
 require_relative "../oasis"
+require_relative "browser_agent"
 require_relative "data_parser_utils"
 require_relative "data_parser"
 require_relative "data_part_parser"
@@ -7,24 +7,47 @@ require_relative "data_part_parser"
 module Relaton
   module Oasis
     class DataFetcher < Core::DataFetcher
+      STANDARDS_URL = "https://www.oasis-open.org/standards/".freeze
+      RETRIABLE_ERRORS = [
+        SocketError,
+        Ferrum::TimeoutError,
+        Ferrum::PendingConnectionsError,
+        Ferrum::StatusError,
+      ].freeze
+
       def log_error(msg)
         Util.error msg
       end
 
       def fetch(_source = nil)
-        agent = Mechanize.new
-        resp = agent.get "https://www.oasis-open.org/standards/"
-        doc = Nokogiri::HTML resp.body
+        doc = with_retry { agent.get(STANDARDS_URL) }
         doc.xpath("//details").map do |item|
-          save_doc DataParser.new(item, @errors).parse
+          save_doc DataParser.new(item, @errors, agent: agent).parse
           fetch_parts item
         end
         index.save
         report_errors
+      ensure
+        @agent&.quit
       end
 
       private
 
+      def agent
+        @agent ||= BrowserAgent.new
+      end
+
+      def with_retry
+        tries = 0
+        begin
+          tries += 1
+          yield
+        rescue *RETRIABLE_ERRORS => e
+          retry if tries < 4
+          raise e
+        end
+      end
+
       def index
         @index ||= Relaton::Index.find_or_create(
           :oasis, file: "#{INDEXFILE}.yaml"
@@ -38,7 +61,9 @@ module Relaton
         parts = item.xpath(xpath)
         return unless parts.size > 1
 
-        parts.each { |part| save_doc DataPartParser.new(part, @errors).parse }
+        parts.each do |part|
+          save_doc DataPartParser.new(part, @errors, agent: agent).parse
+        end
       end
 
       def save_doc(doc) # rubocop:disable Metrics/AbcSize

data/lib/relaton/oasis/data_parser.rb

@@ -9,9 +9,10 @@ module Relaton
       #
       # @param [Nokogiri::HTML::Element] node document node
       #
-      def initialize(node, errors = {})
+      def initialize(node, errors = {}, agent: nil)
         @node = node
         @errors = errors
+        @agent = agent
       end
 
       def title
@@ -158,7 +159,7 @@ module Relaton
         rels = @node.xpath(xpath)
         result = if rels.size > 1
                    rels.map do |r|
-                     docid = DataPartParser.new(r).parse_docid
+                     docid = DataPartParser.new(r, @errors, agent: @agent).parse_docid
                      bib = ItemData.new(formattedref: Bib::Formattedref.new(content: docid[0].content))
                      Bib::Relation.new(type: "hasPart", bibitem: bib)
                    end

data/lib/relaton/oasis/data_parser_utils.rb

@@ -1,7 +1,17 @@
+require "mechanize"
+require_relative "browser_agent"
+
 module Relaton
   module Oasis
     # Common methods for document and part parsers.
     module DataParserUtils
+      RETRIABLE_PAGE_ERRORS = [
+        Errno::ETIMEDOUT,
+        Net::OpenTimeout,
+        Ferrum::TimeoutError,
+        Ferrum::PendingConnectionsError,
+        Ferrum::StatusError,
+      ].freeze
       #
       # Parse contributor.
       #
@@ -50,9 +60,17 @@ module Relaton
       def page
         return @page if defined? @page
 
-        if link_node && link_node[:href].match?(/\.html$/)
+        @page = nil
+        return @page unless link_node && link_node[:href].match?(/\.html$/)
+
+        if @agent
+          doc = retry_page(link_node[:href], @agent)
+          @page = doc if doc && @agent.last_status == 200
+        else
+          # No injected agent (e.g. unit tests with VCR cassettes): fall back
+          # to a Mechanize request — VCR can intercept it.
           agent = Mechanize.new
-          agent.agent.allowed_error_codes = [404]
+          agent.agent.allowed_error_codes = [403, 404, 503]
           resp = retry_page(link_node[:href], agent)
           @page = resp if resp && resp.code == "200"
         end
@@ -62,15 +80,15 @@ module Relaton
       # Retry to get page.
       #
       # @param [String] url page URL
-      # @param [Mechanize] agent HTTP client
+      # @param [#get] agent HTTP client responding to #get(url)
       # @param [Integer] retries number of retries
       #
-      # @return [Mechanize::Page, nil] page or nil
+      # @return [Nokogiri::HTML::Document, Mechanize::Page, nil] page or nil
       #
       def retry_page(url, agent, retries = 3)
         sleep 1 # to avoid 429 error
         agent.get url
-      rescue Errno::ETIMEDOUT, Net::OpenTimeout => e
+      rescue *RETRIABLE_PAGE_ERRORS => e
         retry if (retries -= 1).positive?
         Util.error "Failed to get page `#{url}`\n#{e.message}"
         nil
@@ -84,7 +102,7 @@ module Relaton
                            "[starts-with(., 'Editor')]]"
                    page.xpath(xpath).map do |p|
                      create_contribution_info(p, "editor", ["Chair"])
-                   end
+                   end.compact
                  else
                    []
                  end
@@ -101,7 +119,7 @@ module Relaton
                            "[contains(@class, 'Title')]]"
                    page.xpath(xpath).map do |p|
                      create_contribution_info(p, "editor")
-                   end
+                   end.compact
                  else
                    parse_editors_from_text
                  end
@@ -111,6 +129,9 @@ module Relaton
 
       def create_contribution_info(person_node, type, description = [])
         name = person_node.text.match(/^[^(]+/).to_s.strip
+        return nil if name.empty? || !name.match?(/\A\p{L}/) ||
+                      name.match?(%r{\A(?:https?://|urn:)})
+
         email, org = person_node.xpath ".//a[@href]"
         entity = create_person name, email, org
         desc = description.map { |d| Bib::LocalizedMarkedUpString.new(content: d) }
@@ -137,7 +158,15 @@ module Relaton
           [href.split(":")[1]]
         elsif (cf_email = email.at(".//span[@data-cfemail]"))
           decoded = decode_cf_email(cf_email["data-cfemail"])
-          decoded.empty? ? [] : [decoded]
+          return [] if decoded.empty?
+
+          # Cloudflare obfuscates ASCII email characters in the data-cfemail
+          # span but leaves non-ASCII characters (e.g. the Latin "fl" ligature
+          # U+FB02) as plain text outside the span. Concatenate any sibling
+          # text and NFKC-normalize so ligatures become their ASCII equivalent.
+          prefix = cf_email.xpath("./preceding-sibling::node()").map(&:text).join
+          suffix = cf_email.xpath("./following-sibling::node()").map(&:text).join
+          [(prefix + decoded + suffix).unicode_normalize(:nfkc)]
         else
           []
         end

data/lib/relaton/oasis/data_part_parser.rb

@@ -9,9 +9,10 @@ module Relaton
       #
       # @param [Nokogiri::HTML::Element] node document node
       #
-      def initialize(node, errors = {})
+      def initialize(node, errors = {}, agent: nil)
         @node = node
         @errors = errors
+        @agent = agent
       end
 
       def text
@@ -34,11 +35,11 @@ module Relaton
         xpath = "./span[@class='citationTitle' " \
                 "or @class='citeTitle']|./em|./i"
         t = @node.at(xpath)
-        @title = if t then t.text
+        @title = if t
+                   t.text
                  else
-                   text.match(
-                     /(?<content>.+)\s(?:Edited|\d{2}\s\w+\d{4})/,
-                   )[:content]
+                   m = text.match(/(?<content>.+)\s(?:Edited|\d{2}\s\w+\d{4})/)
+                   m ? m[:content] : text
                  end.strip
       end
 
@@ -82,7 +83,7 @@ module Relaton
       # @return [String] document number
       #
       def parse_docnumber
-        ref = @node.at("./span[@class='citationLabel']/strong|./strong|b/span")
+        ref = @node.at("./span/strong|./strong|./b/span")
         num = ref.text.match(/[^\[\]]+/).to_s
         id = parse_errata(num)
         # some part refs need "Pt" to distinguish from root doc
@@ -110,8 +111,8 @@ module Relaton
       # @return [Array<Bib::Date>] bibliographic dates
       #
       def parse_date
-        /(?<on>\d{1,2}\s\w+\s\d{4})/ =~ text
-        result = [Bib::Date.new(at: Date.parse(on).to_s, type: "issued")]
+        match = text.match(/(?<on>\d{1,2}\s\w+\s\d{4})/)
+        result = match ? [Bib::Date.new(at: Date.parse(match[:on]).to_s, type: "issued")] : []
         @errors[:part_date] &&= result.empty?
         result
       end
@@ -182,7 +183,7 @@ module Relaton
       # @return [Array<Bib::Relation>] document relations
       #
       def parse_relation
-        parser = DataParser.new @node.at("./ancestor::details")
+        parser = DataParser.new(@node.at("./ancestor::details"), @errors, agent: @agent)
         fref = parser.parse_docid[0].content
         bib = ItemData.new(formattedref: Bib::Formattedref.new(content: fref))
         result = [Bib::Relation.new(type: "partOf", bibitem: bib)]

data/lib/relaton/oasis/version.rb

@@ -2,6 +2,6 @@
 
 module Relaton
   module Oasis
-    VERSION = "2.1.1"
+    VERSION = "2.1.2"
   end
 end

data/relaton-oasis.gemspec

@@ -31,6 +31,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
+  spec.add_dependency "ferrum", "~> 0.17"
   spec.add_dependency "mechanize", "~> 2.10"
   spec.add_dependency "multi_json", "~> 1.15.0"
   spec.add_dependency "relaton-bib", "~> 2.1.0"

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: relaton-oasis
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 2.1.2
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-05-12 00:00:00.000000000 Z
+date: 2026-05-14 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: ferrum
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.17'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.17'
 - !ruby/object:Gem::Dependency
   name: mechanize
   requirement: !ruby/object:Gem::Requirement
@@ -110,6 +124,7 @@ files:
 - lib/relaton/oasis/bibdata.rb
 - lib/relaton/oasis/bibitem.rb
 - lib/relaton/oasis/bibliography.rb
+- lib/relaton/oasis/browser_agent.rb
 - lib/relaton/oasis/data_fetcher.rb
 - lib/relaton/oasis/data_parser.rb
 - lib/relaton/oasis/data_parser_utils.rb