relaton-cie 2.1.1 → 2.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4cd9e78a6899ffa70d902a2b00eccf5dcce1d2dc15fffe8925dc9b5dc84650af
-  data.tar.gz: 31cb613d144a7aec4c7f100c116dcdcab4d23e3773c362a4e5129cb35f3dbdef
+  metadata.gz: 905831793252b7a1a462a6cc7e4462d0c90c3a4089894cfd3f991fffa87cbe37
+  data.tar.gz: 4740b925366997bd689ca50a690e86653e15afe78ea190e5a17fcc7c43b4b726
 SHA512:
-  metadata.gz: 59e51836382654d2f186f8f6aaf68a7e74f222d0cf4f1e20894070da6822e0a59766e237754412bce1383530f4a6a730da519fd8a73a47c4a985ba7a8d0704a1
-  data.tar.gz: 72460dcabce2101301e1b55347ca762f88b003c66d4fbd246adba213030fec121def7550f08e132891790885aa21fbddcba1c7caa5f2efbe0c1769c0b1600630
+  metadata.gz: 7ef9caf1b5421a8e0c7a31238f1ccb4767987827df554fc6e0fc24f56be74183e78e5a4f3093d90c0e6fa4060997288b217d15f7e91ad7bb7e2da31cc318587e
+  data.tar.gz: d68a2916f072bfb7113b154c38ba25e96e616fd33fb9094f7d145bdb7aa4bff27c27c02bc5f1b21e8eb21545e9b6e5ab4a89b72f56c877c5663b862281a6a4ef

data/lib/relaton/cie/data_fetcher.rb

@@ -2,7 +2,8 @@
 
 require "English"
 require "fileutils"
-require "mechanize"
+require "ferrum"
+require "nokogiri"
 require "relaton/index"
 require "relaton/bib"
 require "relaton/core/data_fetcher"
@@ -10,22 +11,71 @@ require_relative "../cie"
 
 module Relaton
   module Cie
+    # Thin Ferrum-backed HTTP agent that mimics the Mechanize#get interface
+    # used elsewhere in DataFetcher. Drives headless Chrome with stealth
+    # tweaks so the CIE catalogue host (Cloudflare-protected accuristech)
+    # serves real HTML instead of a "Just a moment..." challenge.
+    class BrowserAgent
+      UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " \
+           "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
+      CHALLENGE_MARKERS = ["Just a moment", "challenge-platform"].freeze
+      MAX_CHALLENGE_WAIT = 30
+
+      def initialize
+        @browser = Ferrum::Browser.new(
+          headless: true,
+          timeout: 90,
+          process_timeout: 90,
+          window_size: [1366, 768],
+          browser_options: {
+            "disable-blink-features" => "AutomationControlled",
+            "disable-quic" => nil,
+            "no-sandbox" => nil
+          }
+        )
+        @browser.headers.set(
+          "Accept-Language" => "en-US,en;q=0.9",
+          "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9," \
+                      "image/webp,*/*;q=0.8",
+          "User-Agent" => UA
+        )
+        # Pre-mask the most common headless-Chrome tells before any page JS runs.
+        @browser.evaluate_on_new_document(<<~JS)
+          Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
+          Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
+          Object.defineProperty(navigator, 'plugins', { get: () => [1,2,3,4,5] });
+          window.chrome = { runtime: {} };
+        JS
+      end
+
+      def get(url)
+        @browser.go_to(url)
+        wait_for_challenge
+        Nokogiri::HTML(@browser.body)
+      end
+
+      def quit
+        @browser&.quit
+      ensure
+        @browser = nil
+      end
+
+      private
+
+      def wait_for_challenge
+        MAX_CHALLENGE_WAIT.times do
+          return unless CHALLENGE_MARKERS.any? { |m| @browser.body.include?(m) }
+
+          sleep 1
+        end
+      end
+    end
+
     class DataFetcher < Relaton::Core::DataFetcher
       URL = "https://www.techstreet.com/cie/searches/31156444?page=1&per_page=100"
 
       def agent
-        return @agent if @agent
-
-        @agent = Mechanize.new
-        @agent.request_headers = {
-          "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
-          "Accept-Language" => "en-US,en;q=0.5",
-          "Connection" => "keep-alive",
-          "sec-ch-ua" => '"Chromium";v="91", "Google Chrome";v="91", ";Not A Brand";v="99"',
-          "Sec-Fetch-Dest" => "document"
-        }
-        @agent.user_agent_alias = "Linux Firefox"
-        @agent
+        @agent ||= BrowserAgent.new
       end
 
       def index
@@ -263,6 +313,8 @@ module Relaton
       def fetch(_source = nil)
         fetch_doc
         report_errors
+      ensure
+        @agent&.quit
       end
 
       def fetch_doc(url = URL)
@@ -270,19 +322,28 @@ module Relaton
         result.xpath("//li[@data-product]").each { |hit| parse_page hit }
         np = result.at '//a[@class="next_page"]'
         if np
-          fetch_doc "https://www.techstreet.com#{np[:href]}"
+          next_href = np[:href]
+          next_url = next_href.start_with?("http") ? next_href : "https://www.techstreet.com#{next_href}"
+          fetch_doc next_url
         else
           index.save
         end
       end
 
+      RETRIABLE_ERRORS = [
+        SocketError,
+        Ferrum::TimeoutError,
+        Ferrum::PendingConnectionsError,
+        Ferrum::StatusError
+      ].freeze
+
       def time_req
         tries = 0
         begin
           tries += 1
           sleep [4 - (Time.now - @last_request_time).to_i, 0].max if @last_request_time
           yield
-        rescue SocketError => e
+        rescue *RETRIABLE_ERRORS => e
           retry if tries < 4
           raise e
         ensure

data/lib/relaton/cie/version.rb

@@ -1,5 +1,5 @@
 module Relaton
   module Cie
-    VERSION = "2.1.1".freeze
+    VERSION = "2.1.2".freeze
   end
 end

data/relaton-cie.gemspec

@@ -29,6 +29,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
+  spec.add_dependency "ferrum", "~> 0.17"
   spec.add_dependency "mechanize", "~> 2.10"
   spec.add_dependency "parslet", "~> 2.0.0"
   spec.add_dependency "relaton-bib", "~> 2.1.0"

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: relaton-cie
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 2.1.2
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-05-12 00:00:00.000000000 Z
+date: 2026-05-14 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: ferrum
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.17'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.17'
 - !ruby/object:Gem::Dependency
   name: mechanize
   requirement: !ruby/object:Gem::Requirement