scraper-central-ruby 2.1.0 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 344375e9baae6533bc98bb11d109dbe8771592d0d7029a83f3ee27176981938c
-  data.tar.gz: 1d02478a129bc7641c74733e3e4de6495112a2151ba6b2802fa1cf0a44f71c04
+  metadata.gz: 5c81c9909df9cbc05523c066e027df4aff98182c1b59f6e554e73ffed5bfbed1
+  data.tar.gz: 65254df4076335ac2df67f95a0f68f779d8f9eac6ee2d97232cd8b6dfb69556b
 SHA512:
-  metadata.gz: 0f6bcada267645f395aaf66aa6e39062beec893b2c3734be037b10cc7fda6664a51e37ab2de65c35cfc3a9786154b85955402f5c82f4934af844d99c202276e1
-  data.tar.gz: 60a335b4d9e62b53d98cde2436e522f0d93598251a64527439f7dd85576c612afeac5490ec81429f271c6eabd73f20872e299076abcef83606bae0c027ffe91c
+  metadata.gz: ed50acebd0f138ed65d8d354d3f8b71c0558d6720d1bae845831f712d8abf0d4b6f4e0e770720fb235aa469a37703cc47b27596060b9eb5fab23b82c0c3eadfe
+  data.tar.gz: f040bec9965ae7c6bd6c20126cf76a20ad9aa3c84481df20f598c0f46f1fac0eba9f1b35cd4570c014a3527e645eec481b9ecf8c01d39e6eaa19b6d0889349af

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    scraper-central-ruby (2.1.0)
+    scraper-central-ruby (2.2.0)
       activesupport (~> 7.0)
       brotli (~> 0.5.0)
       net-http-persistent (~> 4.0)
@@ -43,6 +43,7 @@ GEM
 
 PLATFORMS
   arm64-darwin-23
+  arm64-darwin-25
 
 DEPENDENCIES
   brotli (~> 0.5.0)

data/README.md

@@ -7,7 +7,7 @@ Ruby library to scrape and cache the data
 Add gem `scraper-central-ruby` into Gemfile:
 
 ```bash
-gem 'scraper-central-ruby', git: 'git@github.com:patterninc/scraper-central-ruby.git', tag: 'v2.1.0'
+gem 'scraper-central-ruby', git: 'git@github.com:patterninc/scraper-central-ruby.git', tag: 'v2.2.0'
 ```
 
 ```bash
@@ -143,6 +143,15 @@ puts "Status Code: ", response.code
 puts "Headers: ", response.headers
 ```
 
+### Single-roundtrip get-dom
+
+Use `enable_get_dom = true` to call `/v1/get-dom` on the scraper-central server. On cache miss the server scrapes and returns HTML in one request (no client-side proxy or put-cache). Reuses `SERVER_URL_PUT_CACHE` when set, otherwise `SERVER_URL_GET_CACHE`. `cache_duration` is required (`age` in the request payload).
+
+```ruby
+scraper_central.enable_get_dom = true
+response = scraper_central.fetch("https://www.example-brand.com/product")
+```
+
 ## Documentation
 
 ### Configuration Functions
@@ -153,6 +162,7 @@ puts "Headers: ", response.headers
 - `scraper_central.timeout=`: Sets the request timeout in seconds.
 - `scraper_central.tls_verify=`: Configures TLS verification.
 - `scraper_central.enable_image_cache=` Enable/Disable image caching by setting true/false
+- `scraper_central.enable_get_dom=`: Fetches via `/v1/get-dom` for synchronous cache-or-scrape in one roundtrip.
 ### Proxy Methods
 
 - `scraper_central.query_params=`: Sets query parameters to be appended to each request URL.

data/lib/cache_server.rb

@@ -9,6 +9,10 @@ require 'auth'
 class CacheServer
   AUTH0_BEARER_HEADER = 'Authorization'.freeze
   CONTENT_TYPE_HEADER = 'Content-Type'.freeze
+  GET_CACHE_ENDPOINT = '/v1/get-cache'.freeze
+  GET_DOM_ENDPOINT = '/v1/get-dom'.freeze
+  PUT_CACHE_ENDPOINT = '/v1/put-cache'.freeze
+  HTTP_OK = 200
 
   # Persistent HTTP clients shared across all instances — reuses TCP connections
   # instead of opening a new connection on every request.
@@ -26,35 +30,64 @@ class CacheServer
     @s3_key = args[:s3_key]
     @enable_image_cache = args[:enable_image_cache]
     @auth_config = args[:auth_config]
+    @use_get_dom = args[:use_get_dom]
     @logger = Logger.new($stdout)
   end
 
   def get_cache(url)
+    base_url = ENV['SERVER_URL_GET_CACHE']
+    endpoint = GET_CACHE_ENDPOINT
+    if @use_get_dom
+      put_cache = ENV['SERVER_URL_PUT_CACHE']
+      base_url = put_cache.to_s != '' ? put_cache : ENV['SERVER_URL_GET_CACHE']
+      endpoint = GET_DOM_ENDPOINT
+      if base_url.nil? || base_url.empty?
+        raise StandardError, 'SERVER_URL_PUT_CACHE or SERVER_URL_GET_CACHE must be set for get-dom'
+      end
+    end
+
     payload = prepare_get_cache_payload(url)
 
-    uri = URI.parse("#{ENV['SERVER_URL_GET_CACHE']}/v1/get-cache")
+    uri = URI.parse("#{base_url}#{endpoint}")
     request = Net::HTTP::Get.new(uri.request_uri, auth_headers)
     request.body = payload.to_json
 
     begin
       response = GET_HTTP.request(uri, request)
 
+      response_code = response.code&.to_i
+      if @use_get_dom && response_code != HTTP_OK
+        raise StandardError, "get-dom returned status #{response.code}: #{response.body}"
+      end
+
       if response.content_type.include?('application/json')
         response_body = JSON.parse(response.body)
-        return '', nil, proxy_from_server(response_body) if response_body.key?('proxyUrl')
+        return '', nil, proxy_from_server(response_body), 200 if response_body.key?('proxyUrl')
         page_from_server = response_body['body']
         if @enable_image_cache
           decoded_data = Base64.decode64(page_from_server)
           page_from_server = StringIO.new(decoded_data)
         end
-        return page_from_server, headers_from_server(response_body), nil
+        status_code = response_body['statusCode'] || 200
+        return page_from_server, headers_from_server(response_body), nil, status_code
+      elsif @use_get_dom
+        raise StandardError, "get-dom returned unexpected content type #{response.content_type.inspect}"
       else
         @logger.error "Unexpected response type: #{response.content_type}, body: #{response.body}, code: #{response.code}"
       end
+    rescue JSON::ParserError => e
+      if @use_get_dom
+        raise StandardError, "error unmarshaling get-dom response: #{e.message}"
+      end
+      @logger.error "Error sending request to server: #{e.message}"
     rescue StandardError => e
+      if @use_get_dom
+        @logger.error "Error sending request to server: #{e.message}"
+        raise
+      end
       @logger.error "Error sending request to server: #{e.message}"
     end
-    ['', nil, nil]
+    ['', nil, nil, 200]
   end
 
   def put_cache(cache_key, page, headers, cookies, enable_image_cache)
@@ -66,13 +99,13 @@ class CacheServer
       enableImageCache: enable_image_cache
     }
 
-    uri = URI.parse("#{ENV['SERVER_URL_PUT_CACHE']}/v1/put-cache")
+    uri = URI.parse("#{ENV['SERVER_URL_PUT_CACHE']}#{PUT_CACHE_ENDPOINT}")
     request = Net::HTTP::Post.new(uri.request_uri, auth_headers)
     request.body = payload.to_json
 
     begin
       response = PUT_HTTP.request(uri, request)
-      if response.code.to_i != 200
+      if response.code&.to_i != HTTP_OK
         error_message = "Server returned bad status: #{response.code}"
         @logger.error error_message
         raise StandardError, error_message

data/lib/scraper_central/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class ScraperCentral
-  VERSION = '2.1.0'
+  VERSION = '2.2.0'
 end

data/lib/scraper_central.rb

@@ -9,7 +9,7 @@ require 'proxy/scraper_api'
 
 class ScraperCentral
   attr_accessor :cache_duration, :proxy_name, :s3_key, :enable_js, :tls_verify, :headers, :query_params, :cookies,
-                :timeout, :retry_attr, :enable_image_cache, :auth_config
+                :timeout, :retry_attr, :enable_image_cache, :auth_config, :enable_get_dom
 
   def initialize
     @lock = Mutex.new
@@ -19,10 +19,11 @@ class ScraperCentral
   def fetch(url)
     @lock.synchronize do
       @url = url
-      page_from_server, headers_from_server, proxy_from_server = cache_server.get_cache(@url)
+      page_from_server, headers_from_server, proxy_from_server, status_code = cache_server.get_cache(@url)
       if proxy_from_server.nil?
         print_proxy_values
-        return Response.new(code: 200, body: page_from_server, headers: headers_from_server)
+        response_code = enable_get_dom ? status_code : 200
+        return Response.new(code: response_code, body: page_from_server, headers: headers_from_server)
       else
         proxy_response = nil
 
@@ -53,12 +54,14 @@ class ScraperCentral
           return Response.new(code: status_code)
         end
 
-        Thread.new do
-          cache_server.put_cache(proxy_from_server['cacheKey'], proxy_response.body, proxy_response.headers,
-                                 proxy_response.cookies, enable_image_cache)
-          @logger.info("Cache successfully sent to server key: #{proxy_from_server['cacheKey']}")
-        rescue StandardError => e
-          @logger.error("Error uploading cache to server key: #{proxy_from_server['cacheKey']}, error: #{e.message}")
+        unless enable_get_dom
+          Thread.new do
+            cache_server.put_cache(proxy_from_server['cacheKey'], proxy_response.body, proxy_response.headers,
+                                   proxy_response.cookies, enable_image_cache)
+            @logger.info("Cache successfully sent to server key: #{proxy_from_server['cacheKey']}")
+          rescue StandardError => e
+            @logger.error("Error uploading cache to server key: #{proxy_from_server['cacheKey']}, error: #{e.message}")
+          end
         end
 
         print_proxy_values
@@ -98,7 +101,8 @@ class ScraperCentral
       cache_duration: cache_duration,
       s3_key: s3_key,
       enable_image_cache: enable_image_cache,
-      auth_config: auth_config
+      auth_config: auth_config,
+      use_get_dom: enable_get_dom
     }
     CacheServer.new(args)
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scraper-central-ruby
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.2.0
 platform: ruby
 authors:
 - Patterninc
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-04-23 00:00:00.000000000 Z
+date: 2026-06-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: brotli