relaton-cie 2.1.1 → 2.2.0.pre.alpha.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4cd9e78a6899ffa70d902a2b00eccf5dcce1d2dc15fffe8925dc9b5dc84650af
-  data.tar.gz: 31cb613d144a7aec4c7f100c116dcdcab4d23e3773c362a4e5129cb35f3dbdef
+  metadata.gz: 23dae19e6d6a0c33a0cc45fa13849bb2d1cb7c2df5edc6c0f4e63d9ef7cdc019
+  data.tar.gz: 6a69081acd7bc15619eee05e1c272b9ca7a4cd40e81ead45621e6e4d38af4dcc
 SHA512:
-  metadata.gz: 59e51836382654d2f186f8f6aaf68a7e74f222d0cf4f1e20894070da6822e0a59766e237754412bce1383530f4a6a730da519fd8a73a47c4a985ba7a8d0704a1
-  data.tar.gz: 72460dcabce2101301e1b55347ca762f88b003c66d4fbd246adba213030fec121def7550f08e132891790885aa21fbddcba1c7caa5f2efbe0c1769c0b1600630
+  metadata.gz: 2d5345f1337ab5cdb9e7b47c9de21fd698dc3bbe9825a463367f8e05ca3dcf9cdd08615939154db886e62aec85145660362c018ed1ba4ce5ec72bdf47b986795
+  data.tar.gz: 84029b0877a63e2408e053d361740b3c2ba7cab9f0a24a2ef67cbb8d14e4c08e88dbbcc1177d5a19805862255b0ec815318c7bbbef1ee8608e1bb42723d72902

data/Gemfile

@@ -3,6 +3,14 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in relaton_cie.gemspec
 gemspec
 
+# Use local monorepo sibling gems where available.
+Dir["../*/"].each do |dir|
+  name = File.basename(dir)
+  next if name == File.basename(__dir__)
+  next unless File.exist?(File.join(dir, "#{name}.gemspec"))
+  gem name, path: dir
+end
+
 
 gem "equivalent-xml", "~> 0.6"
 gem "pry-byebug"

data/lib/relaton/cie/data_fetcher.rb

@@ -2,7 +2,8 @@
 
 require "English"
 require "fileutils"
-require "mechanize"
+require "ferrum"
+require "nokogiri"
 require "relaton/index"
 require "relaton/bib"
 require "relaton/core/data_fetcher"
@@ -10,22 +11,71 @@ require_relative "../cie"
 
 module Relaton
   module Cie
+    # Thin Ferrum-backed HTTP agent that mimics the Mechanize#get interface
+    # used elsewhere in DataFetcher. Drives headless Chrome with stealth
+    # tweaks so the CIE catalogue host (Cloudflare-protected accuristech)
+    # serves real HTML instead of a "Just a moment..." challenge.
+    class BrowserAgent
+      UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " \
+           "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
+      CHALLENGE_MARKERS = ["Just a moment", "challenge-platform"].freeze
+      MAX_CHALLENGE_WAIT = 30
+
+      def initialize
+        @browser = Ferrum::Browser.new(
+          headless: true,
+          timeout: 90,
+          process_timeout: 90,
+          window_size: [1366, 768],
+          browser_options: {
+            "disable-blink-features" => "AutomationControlled",
+            "disable-quic" => nil,
+            "no-sandbox" => nil
+          }
+        )
+        @browser.headers.set(
+          "Accept-Language" => "en-US,en;q=0.9",
+          "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9," \
+                      "image/webp,*/*;q=0.8",
+          "User-Agent" => UA
+        )
+        # Pre-mask the most common headless-Chrome tells before any page JS runs.
+        @browser.evaluate_on_new_document(<<~JS)
+          Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
+          Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
+          Object.defineProperty(navigator, 'plugins', { get: () => [1,2,3,4,5] });
+          window.chrome = { runtime: {} };
+        JS
+      end
+
+      def get(url)
+        @browser.go_to(url)
+        wait_for_challenge
+        Nokogiri::HTML(@browser.body)
+      end
+
+      def quit
+        @browser&.quit
+      ensure
+        @browser = nil
+      end
+
+      private
+
+      def wait_for_challenge
+        MAX_CHALLENGE_WAIT.times do
+          return unless CHALLENGE_MARKERS.any? { |m| @browser.body.include?(m) }
+
+          sleep 1
+        end
+      end
+    end
+
     class DataFetcher < Relaton::Core::DataFetcher
       URL = "https://www.techstreet.com/cie/searches/31156444?page=1&per_page=100"
 
       def agent
-        return @agent if @agent
-
-        @agent = Mechanize.new
-        @agent.request_headers = {
-          "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
-          "Accept-Language" => "en-US,en;q=0.5",
-          "Connection" => "keep-alive",
-          "sec-ch-ua" => '"Chromium";v="91", "Google Chrome";v="91", ";Not A Brand";v="99"',
-          "Sec-Fetch-Dest" => "document"
-        }
-        @agent.user_agent_alias = "Linux Firefox"
-        @agent
+        @agent ||= BrowserAgent.new
       end
 
       def index
@@ -263,6 +313,8 @@ module Relaton
       def fetch(_source = nil)
         fetch_doc
         report_errors
+      ensure
+        @agent&.quit
       end
 
       def fetch_doc(url = URL)
@@ -270,19 +322,28 @@ module Relaton
         result.xpath("//li[@data-product]").each { |hit| parse_page hit }
         np = result.at '//a[@class="next_page"]'
         if np
-          fetch_doc "https://www.techstreet.com#{np[:href]}"
+          next_href = np[:href]
+          next_url = next_href.start_with?("http") ? next_href : "https://www.techstreet.com#{next_href}"
+          fetch_doc next_url
         else
           index.save
         end
       end
 
+      RETRIABLE_ERRORS = [
+        SocketError,
+        Ferrum::TimeoutError,
+        Ferrum::PendingConnectionsError,
+        Ferrum::StatusError
+      ].freeze
+
       def time_req
         tries = 0
         begin
           tries += 1
           sleep [4 - (Time.now - @last_request_time).to_i, 0].max if @last_request_time
           yield
-        rescue SocketError => e
+        rescue *RETRIABLE_ERRORS => e
           retry if tries < 4
           raise e
         ensure

data/lib/relaton/cie/version.rb

@@ -1,5 +1,5 @@
 module Relaton
   module Cie
-    VERSION = "2.1.1".freeze
+    VERSION = "2.2.0.pre.alpha.1".freeze
   end
 end

data/relaton-cie.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |spec|
   DESCRIPTION
   spec.homepage      = "https://github.com/metanorma/relaton-cie"
   spec.license       = "BSD-2-Clause"
-  spec.required_ruby_version = Gem::Requirement.new(">= 3.2.0")
+  spec.required_ruby_version = Gem::Requirement.new(">= 3.3.0")
 
   spec.metadata["homepage_uri"] = spec.homepage
 
@@ -29,9 +29,9 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
+  spec.add_dependency "ferrum", "~> 0.17"
   spec.add_dependency "mechanize", "~> 2.10"
-  spec.add_dependency "parslet", "~> 2.0.0"
-  spec.add_dependency "relaton-bib", "~> 2.1.0"
-  spec.add_dependency "relaton-core", "~> 0.0.12"
-  spec.add_dependency "relaton-index", "~> 0.2.0"
+  spec.add_dependency "relaton-bib", "~> 2.2.0.pre.alpha.1"
+  spec.add_dependency "relaton-core", "~> 2.2.0.pre.alpha.1"
+  spec.add_dependency "relaton-index", "~> 2.2.0.pre.alpha.1"
 end

metadata

@@ -1,85 +1,85 @@
 --- !ruby/object:Gem::Specification
 name: relaton-cie
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 2.2.0.pre.alpha.1
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-05-12 00:00:00.000000000 Z
+date: 2026-06-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: mechanize
+  name: ferrum
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.10'
+        version: '0.17'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.10'
+        version: '0.17'
 - !ruby/object:Gem::Dependency
-  name: parslet
+  name: mechanize
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: '2.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: '2.10'
 - !ruby/object:Gem::Dependency
   name: relaton-bib
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: 2.2.0.pre.alpha.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: 2.2.0.pre.alpha.1
 - !ruby/object:Gem::Dependency
   name: relaton-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.12
+        version: 2.2.0.pre.alpha.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.12
+        version: 2.2.0.pre.alpha.1
 - !ruby/object:Gem::Dependency
   name: relaton-index
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 2.2.0.pre.alpha.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 2.2.0.pre.alpha.1
 description: |
   Relaton::Cie: retrieve CIE Standards for bibliographic use
   using the BibliographicItem model.
@@ -93,7 +93,6 @@ files:
 - ".github/workflows/release.yml"
 - ".gitignore"
 - ".rspec"
-- ".rubocop.yml"
 - CLAUDE.md
 - Gemfile
 - LICENSE.txt
@@ -102,11 +101,6 @@ files:
 - bin/console
 - bin/rspec
 - bin/setup
-- grammars/basicdoc.rng
-- grammars/biblio-standoc.rng
-- grammars/biblio.rng
-- grammars/relaton-cie-compile.rng
-- grammars/relaton-cie.rng
 - lib/relaton/cie.rb
 - lib/relaton/cie/bibdata.rb
 - lib/relaton/cie/bibitem.rb
@@ -133,7 +127,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.2.0
+      version: 3.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="

data/.rubocop.yml

@@ -1,12 +0,0 @@
-# This project follows the Ribose OSS style guide.
-# https://github.com/riboseinc/oss-guides
-# All project-specific additions and overrides should be specified in this file.
-
-require: rubocop-rails
-
-inherit_from:
-  - https://raw.githubusercontent.com/riboseinc/oss-guides/master/ci/rubocop.yml
-AllCops:
-  TargetRubyVersion: 3.2
-Rails:
-  Enabled: false