scraper-central-ruby 1.0.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3a80d1ba8b02b21d51a048f7b153f04e23176ffcd5fc258f004ac903776fe831
-  data.tar.gz: 9982839e9a725452a3f26022ccbd46eabea0aa8ac3b94fd1e4bdcda172ae58cc
+  metadata.gz: 4a17d4043d0b53c2a19fe4629d316b3fcc3cb04c4bfd640555449121d13c034b
+  data.tar.gz: 8c98a99a53644ebf9e19e337b15b25c2e5887c85243f9b3c4fd2a949bf2acafa
 SHA512:
-  metadata.gz: 20f9efb238092af60d758971d8a84752a1cf24b9f927795352be2d66fc47784b3050b629300e0c7ed9b8baa04521970600851ce44d3a2b592f556a489a35ae5f
-  data.tar.gz: c464c0c5a20b58366f2e738aee1cabb1b66555c3d9217b277fe9d9ab3433968f596be12dad4717738017b725b9044e3db0e24797aeec146d93d3abe1527119ce
+  metadata.gz: e3426ffc94e2fd97fa80b055a0a6c2db8ee326dbcb163c33d3007278a1aaf71def15806126bc8ce0e44ae0562075d04f273cc73fe1ceec6bcc1d71d285387088
+  data.tar.gz: 888fc81bf4dadbc686c7819833eb46c64f390d11474afb5ea36dcd1a4ce9dfdabd7a56f0684c26d7c0a0b39a3b2d399c57d0637b3383e6d3f1bd5f46cd96966e

data/.github/pull-request-template.md

@@ -0,0 +1,24 @@
+## PR Details
+Clickup Link - 
+
+### Description 
+
+#### Types of changes
+
+<!--- What types of changes does your code introduce? Put an `x` in all the boxes that apply: -->
+
+- [ ] Docs change / refactoring / dependency upgrade
+- [ ] Bug fix (non-breaking change which fixes an issue)
+- [ ] New feature (non-breaking change which adds functionality)
+- [ ] Breaking change (fix or feature that would cause existing functionality to change)
+
+#### Checklist
+
+<!--- Go over all the following points, and put an `x` in all the boxes that apply. -->
+<!--- If you're unsure about any of these, don't hesitate to ask. We're here to help! -->
+
+- [ ] My code follows the code style of this project.
+  <!--- Ruby style guide https://github.com/rubocop/ruby-style-guide -->
+  <!--- Go Style Guide https://github.com/uber-go/guide/blob/master/style.md -->
+- [ ] My change requires a change to the documentation and I have updated the documentation accordingly.
+- [ ] I have added tests to cover my changes.

data/Gemfile.lock

@@ -1,17 +1,48 @@
 PATH
   remote: .
   specs:
-    scraper-central-ruby (1.0.0)
+    scraper-central-ruby (2.0.0)
+      activesupport (~> 7.0)
+      brotli (~> 0.5.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    activesupport (7.1.5.1)
+      base64
+      benchmark (>= 0.3)
+      bigdecimal
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      connection_pool (>= 2.2.5)
+      drb
+      i18n (>= 1.6, < 2)
+      logger (>= 1.4.2)
+      minitest (>= 5.1)
+      mutex_m
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0)
+    base64 (0.2.0)
+    benchmark (0.4.0)
+    bigdecimal (3.1.9)
+    brotli (0.5.0)
+    concurrent-ruby (1.3.5)
+    connection_pool (2.5.0)
+    drb (2.2.1)
+    i18n (1.14.7)
+      concurrent-ruby (~> 1.0)
+    logger (1.6.5)
+    minitest (5.25.4)
+    mutex_m (0.3.0)
     rake (13.2.1)
+    securerandom (0.3.2)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
 
 PLATFORMS
   arm64-darwin-23
 
 DEPENDENCIES
+  brotli (~> 0.5.0)
   bundler (~> 2.4.22)
   rake (~> 13.0)
   scraper-central-ruby!

data/README.md

@@ -7,7 +7,7 @@ Ruby library to scrape and cache the data
 Add gem `scraper-central-ruby` into Gemfile:
 
 ```bash
-gem 'scraper-central-ruby', git: 'git@github.com:patterninc/scraper-central-ruby.git', tag: 'v1.0.0'
+gem 'scraper-central-ruby', git: 'git@github.com:patterninc/scraper-central-ruby.git', tag: 'v2.0.0'
 ```
 
 ```bash
@@ -117,6 +117,22 @@ Fetches the URL through the proxy:
 
 ```ruby
 scraper_central = ScraperCentral.new
+scraper_central.proxy_name = "BrightData"
+scraper_central.cache_duration = 360
+scraper_central.retry_attr = {
+  count: 5
+}
+scraper_central.s3_key = {
+  country: "US",
+  marketplace: "Amazon",
+  page_type: "product-question",
+  identifier: "B0BQZBPS4G",
+  page_number: 1
+}
+scraper_central.auth_config = {
+  client_id: 'client_id token',
+  client_secret: 'client_secret token'
+}
 
 response = scraper_central.fetch("https://example.com")
 
@@ -136,13 +152,18 @@ puts "Headers: ", response.headers
 - `scraper_central.retry_attr=`: Configures retry logic, including the number of attempts, wait time between attempts.
 - `scraper_central.timeout=`: Sets the request timeout in seconds.
 - `scraper_central.tls_verify=`: Configures TLS verification.
-
+- `scraper_central.enable_image_cache=` Enable/Disable image caching by setting true/false
 ### Proxy Methods
 
 - `scraper_central.query_params=`: Sets query parameters to be appended to each request URL.
 - `scraper_central.headers=`: Adds custom headers to requests. (Accept, Accept-Encoding or Content-Type).
 - `scraper_central.cookies=`: Parses a JSON string of cookies and sets them for subsequent requests.
 
+### Auth configurations method
+
+- `scraper_central.auth_config=`: Adds client_id and client_secret to generate auth token in gem and send it in headers while requesting the scraper central apis
+
+
 ### Proxy Methods For S3 Key
 
 - `scraper_central.s3_key=`: Updates the proxy's target country or S3 key structure country, Marketplace, Pre defined names for page views, e.g. `detail-page`. S3 key value for page identifer e.g. ASIN or Product ID. PageNumber if page is paginated.

data/lib/auth.rb

@@ -0,0 +1,46 @@
+require 'net/http'
+require 'uri'
+require 'json'
+require 'openssl'
+require 'cache'
+
+class Auth
+  AUTH0_TOKEN_ENDPOINT = "https://dev-pattern.auth0.com/oauth/token".freeze
+  AUTH0_DEFAULT_AUDIENCE = "https://dev-pattern.auth0.com/api/v2/".freeze
+  AUTH0_DEFAULT_GRANT_TYPE = "client_credentials".freeze
+  AUTH0_CACHE_KEY = 'AUTH0_CACHE_KEY'.freeze
+
+  def initialize(auth_config)
+    @auth_config = auth_config
+  end
+
+  def token_params
+    {
+      grant_type: AUTH0_DEFAULT_GRANT_TYPE,
+      client_id: @auth_config[:client_id],
+      client_secret: @auth_config[:client_secret],
+      audience: AUTH0_DEFAULT_AUDIENCE
+    }
+  end
+
+  def get_token
+    Cache.read(AUTH0_CACHE_KEY) || generate_token
+  end
+
+  def generate_token
+    url = URI(AUTH0_TOKEN_ENDPOINT)
+
+    http = Net::HTTP.new(url.host, url.port)
+    http.use_ssl = true
+    http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+
+    request = Net::HTTP::Post.new(url)
+    request["content-type"] = 'application/json'
+    request.body = token_params.to_json
+
+    response = http.request(request)
+    token = JSON.parse(response.read_body)['access_token']
+    Cache.write(AUTH0_CACHE_KEY, token, expires_in: 1.day)
+    token
+  end
+end

data/lib/cache.rb

@@ -0,0 +1,13 @@
+class Cache
+  def self.store
+    @store ||= ActiveSupport::Cache::MemoryStore.new
+  end
+
+  def self.read(key)
+    store.read(key)
+  end
+
+  def self.write(key, value, expires_in: nil)
+    store.write(key, value, expires_in: expires_in)
+  end
+end

data/lib/cache_server.rb

@@ -4,34 +4,46 @@ require 'net/http'
 require 'uri'
 require 'json'
 require 'logger'
+require 'auth'
 
 class CacheServer
-  def initialize(proxy_name, enable_js, cache_duration, s3_key)
-    @proxy_name = proxy_name
-    @enable_js = enable_js
-    @cache_duration = cache_duration
-    @s3_key = s3_key
+  AUTH0_BEARER_HEADER = 'Authorization'.freeze
+  CONTENT_TYPE_HEADER = 'Content-Type'.freeze
+
+  def initialize(args)
+    @proxy_name = args[:proxy_name]
+    @enable_js = args[:enable_js]
+    @cache_duration = args[:cache_duration]
+    @s3_key = args[:s3_key]
+    @enable_image_cache = args[:enable_image_cache]
+    @auth_config = args[:auth_config]
     @logger = Logger.new($stdout)
   end
 
   def get_cache(url)
     payload = prepare_get_cache_payload(url)
 
-    uri = URI.parse("#{ENV['SERVER_URL_GET_CACHE']}/get-cache")
+    uri = URI.parse("#{ENV['SERVER_URL_GET_CACHE']}/v1/get-cache")
     http = Net::HTTP.new(uri.host, uri.port)
     if uri.scheme == 'https'
       http.use_ssl = true
       http.verify_mode = OpenSSL::SSL::VERIFY_NONE
     end
-    request = Net::HTTP::Post.new(uri.request_uri, 'Content-Type' => 'application/json')
+    request = Net::HTTP::Get.new(uri.request_uri, headers)
     request.body = payload.to_json
 
     begin
       response = http.request(request)
+
       if response.content_type.include?('application/json')
         response_body = JSON.parse(response.body)
         return '', nil, proxy_from_server(response_body) if response_body.key?('proxyUrl')
-        return response_body['body'], headers_from_server(response_body), nil
+        page_from_server = response_body['body']
+        if @enable_image_cache
+          decoded_data = Base64.decode64(page_from_server)
+          page_from_server = StringIO.new(decoded_data)
+        end
+        return page_from_server, headers_from_server(response_body), nil
       else
         @logger.error "Unexpected response type: #{response.content_type}, body: #{response.body}, code: #{response.code}"
       end
@@ -41,22 +53,23 @@ class CacheServer
     ['', nil, nil]
   end
 
-  def put_cache(cache_key, page, headers, cookies)
+  def put_cache(cache_key, page, headers, cookies, enable_image_cache)
     payload = {
       cacheKey: cache_key,
       page: page,
       headers: headers,
-      cookies: cookies
+      cookies: cookies,
+      enableImageCache: enable_image_cache
     }
 
-    uri = URI.parse("#{ENV['SERVER_URL_PUT_CACHE']}/put-cache")
+    uri = URI.parse("#{ENV['SERVER_URL_PUT_CACHE']}/v1/put-cache")
     http = Net::HTTP.new(uri.host, uri.port)
     if uri.scheme == 'https'
       http.use_ssl = true
       http.verify_mode = OpenSSL::SSL::VERIFY_NONE
     end
 
-    request = Net::HTTP::Post.new(uri.request_uri, 'Content-Type' => 'application/json')
+    request = Net::HTTP::Post.new(uri.request_uri, headers)
     request.body = payload.to_json
 
     begin
@@ -74,6 +87,17 @@ class CacheServer
 
   private
 
+  def auth_token
+    Auth.new(@auth_config).get_token
+  end
+
+  def headers
+    {
+      CONTENT_TYPE_HEADER => 'application/json',
+      AUTH0_BEARER_HEADER => "Bearer #{auth_token}"
+    }
+  end
+
   def headers_from_server(response_body)
     headers = {}
     if response_body['headers'].is_a?(Hash)
@@ -102,7 +126,8 @@ class CacheServer
       marketplace: @s3_key[:marketplace],
       pageType: @s3_key[:page_type],
       identifier: @s3_key[:identifier],
-      pageNumber: @s3_key[:page_number]
+      pageNumber: @s3_key[:page_number],
+      enableImageCache: @enable_image_cache
     }
   end
 end

data/lib/proxy/base.rb

@@ -18,6 +18,7 @@ module Proxy
       @tls_verify = params.fetch(:tls_verify, true)
       @retry = params[:retry_attr] || {}
       @enable_js = params.fetch(:enable_js, false)
+      @enable_image_cache = params[:enable_image_cache] || false
       @logger = Logger.new($stdout)
     end
 
@@ -35,14 +36,35 @@ module Proxy
     end
 
     def format_response(response)
+      if success_response?(response)
+        if response.header['Content-Encoding'].eql?('gzip')
+          sio = StringIO.new(response.body)
+          gz = Zlib::GzipReader.new(sio)
+          page_content = gz.read()
+        elsif response.header['Content-Encoding'].eql?('br')
+          # Decode response which is in brotli encryption and save it as html
+          page_content = Brotli.inflate(response.body)
+        else
+          page_content = response.body
+        end
+      end
+
       Response.new(
         code: response.code.to_i,
-        body: response.body,
-        headers: response.to_hash,
+        body: page_content,
+        headers: get_headers(response),
         cookies: response.get_fields('set-cookie')
       )
     end
 
+    def get_headers(response)
+      headers = {}
+      response.each_header do |key, value|
+        headers[key] = value
+      end
+      headers
+    end
+
     def prepare_request(uri, proxy_uri = nil)
       http = if proxy_uri.nil?
                Net::HTTP.new(uri.host, uri.port)
@@ -58,5 +80,9 @@ module Proxy
       http.open_timeout = @timeout
       http
     end
+
+    def success_response?(response)
+      response.kind_of?(Net::HTTPSuccess) || response.code.to_i == 200
+    end
   end
 end

data/lib/proxy/crawl_base.rb

@@ -5,22 +5,14 @@ require 'proxy/base'
 module Proxy
   class CrawlBase < Proxy::Base
     def fetch(url, proxy_from_server)
-      uri = URI.parse(proxy_from_server['proxyHost'])
-      uri.query = URI.encode_www_form({
+      req_options = {
         'token' => proxy_from_server['proxyToken'],
         proxy_from_server['proxyCountryKey'] => @country,
         'url' => url
-      }.merge(@query_params))
-
-      request = Net::HTTP::Get.new(uri, @headers)
-      http = prepare_request(uri)
-
-      @cookies.each do |cookie|
-        request.add_field('Cookie', "#{cookie[:name]}=#{cookie[:value]}")
-      end
+      }.merge(@query_params)
 
       response = with_retry do
-        http.request(request)
+        HTTParty.get(proxy_from_server['proxyHost'], query: req_options, headers: @headers)
       end
 
       format_response(response)

data/lib/proxy/scraper_api.rb

@@ -5,24 +5,15 @@ require 'proxy/base'
 module Proxy
   class ScraperApi < Proxy::Base
     def fetch(url, proxy_from_server)
-      uri = URI.parse(proxy_from_server['proxyHost'])
-
-      uri.query = URI.encode_www_form({
+      req_options = {
         'api_key' => proxy_from_server['proxyToken'],
         'render' => @enable_js.to_s,
         proxy_from_server['proxyCountryKey'] => @country,
         'url' => url
-      }.merge(@query_params))
-
-      request = Net::HTTP::Get.new(uri, @headers)
-      http = prepare_request(uri)
-
-      @cookies.each do |cookie|
-        request.add_field('Cookie', "#{cookie[:name]}=#{cookie[:value]}")
-      end
+      }.merge(@query_params)
 
       response = with_retry do
-        http.request(request)
+        HTTParty.get(proxy_from_server['proxyHost'], query: req_options, headers: @headers)
       end
 
       format_response(response)

data/lib/scraper_central/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class ScraperCentral
-  VERSION = '1.0.0'
+  VERSION = '2.0.0'
 end

data/lib/scraper_central.rb

@@ -9,7 +9,7 @@ require 'proxy/scraper_api'
 
 class ScraperCentral
   attr_accessor :cache_duration, :proxy_name, :s3_key, :enable_js, :tls_verify, :headers, :query_params, :cookies,
-                :timeout, :retry_attr
+                :timeout, :retry_attr, :enable_image_cache, :auth_config
 
   def initialize
     @lock = Mutex.new
@@ -25,6 +25,7 @@ class ScraperCentral
         return Response.new(code: 200, body: page_from_server, headers: headers_from_server)
       else
         proxy_response = nil
+
         params = {
           country: s3_key[:country],
           headers: headers,
@@ -33,7 +34,8 @@ class ScraperCentral
           timeout: timeout,
           tls_verify: tls_verify,
           retry_attr: retry_attr,
-          enable_js: enable_js
+          enable_js: enable_js,
+          enable_image_cache: enable_image_cache
         }
 
         case proxy_from_server['proxyName']
@@ -45,18 +47,18 @@ class ScraperCentral
           proxy_response = Proxy::ScraperApi.new(params).fetch(@url, proxy_from_server)
         end
 
-        if proxy_response.nil?
-          @logger.error("Error fetching content from proxy: #{proxy_from_server['proxyName']}")
-          return Response.new(code: 500,
-                              body: StandardError.new("Error fetching content from proxy: #{proxy_from_server['proxyName']}"))
+        if proxy_response.nil? || proxy_response&.code != 200
+          status_code = proxy_response&.code || 500
+          @logger.error("Error fetching content from proxy: #{proxy_from_server['proxyName']}, error code: #{status_code}, params: #{s3_key}")
+          return Response.new(code: status_code)
         end
 
         Thread.new do
           cache_server.put_cache(proxy_from_server['cacheKey'], proxy_response.body, proxy_response.headers,
-                                 proxy_response.cookies)
-          @logger.info('Cache successfully sent to server')
+                                 proxy_response.cookies, enable_image_cache)
+          @logger.info("Cache successfully sent to server key: #{proxy_from_server['cacheKey']}")
         rescue StandardError => e
-          @logger.error("Error uploading cache to server: #{e.message}")
+          @logger.error("Error uploading cache to server key: #{proxy_from_server['cacheKey']}, error: #{e.message}")
         end
 
         print_proxy_values
@@ -86,10 +88,19 @@ class ScraperCentral
     @logger.info("cookies: #{cookies}") if cookies
     @logger.info("timeout: #{timeout}") if timeout
     @logger.info("retry_attr: #{retry_attr}") if retry_attr
+    @logger.info("enable_image_cache: #{enable_image_cache}") if enable_image_cache
   end
 
   def cache_server
-    CacheServer.new(proxy_name, enable_js, cache_duration, s3_key)
+    args = {
+      proxy_name: proxy_name,
+      enable_js: enable_js,
+      cache_duration: cache_duration,
+      s3_key: s3_key,
+      enable_image_cache: enable_image_cache,
+      auth_config: auth_config
+    }
+    CacheServer.new(args)
   end
 end
 

data/scraper-central-ruby.gemspec

@@ -14,6 +14,10 @@ Gem::Specification.new do |spec|
   spec.homepage      = 'https://github.com/patterninc/scraper-central-ruby'
   spec.license       = 'MIT'
 
+  # Add the brotli gem dependency here
+  spec.add_dependency 'brotli', '~> 0.5.0'
+  spec.add_dependency 'activesupport', '~> 7.0'
+
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files         = Dir.chdir(File.expand_path(__dir__)) do
@@ -28,5 +32,7 @@ Gem::Specification.new do |spec|
   # bundle _2.4.22_
   spec.add_development_dependency 'bundler', '~> 2.4.22'
   spec.add_development_dependency 'rake', '~> 13.0'
+  spec.add_development_dependency 'brotli', '~> 0.5.0'
+
   # ... (other development dependencies)
 end

metadata

@@ -1,15 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: scraper-central-ruby
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Patterninc
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-07-11 00:00:00.000000000 Z
+date: 2025-02-04 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: brotli
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.0
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '7.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '7.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -38,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: brotli
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.0
 description:
 email:
 - amol.udage@pattern.com
@@ -45,9 +87,12 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/pull-request-template.md"
 - Gemfile
 - Gemfile.lock
 - README.md
+- lib/auth.rb
+- lib/cache.rb
 - lib/cache_server.rb
 - lib/proxy/base.rb
 - lib/proxy/bright_data.rb