scraper-central-ruby 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 3a80d1ba8b02b21d51a048f7b153f04e23176ffcd5fc258f004ac903776fe831
+  data.tar.gz: 9982839e9a725452a3f26022ccbd46eabea0aa8ac3b94fd1e4bdcda172ae58cc
+SHA512:
+  metadata.gz: 20f9efb238092af60d758971d8a84752a1cf24b9f927795352be2d66fc47784b3050b629300e0c7ed9b8baa04521970600851ce44d3a2b592f556a489a35ae5f
+  data.tar.gz: c464c0c5a20b58366f2e738aee1cabb1b66555c3d9217b277fe9d9ab3433968f596be12dad4717738017b725b9044e3db0e24797aeec146d93d3abe1527119ce

data/Gemfile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in scraper-central-ruby.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,20 @@
+PATH
+  remote: .
+  specs:
+    scraper-central-ruby (1.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    rake (13.2.1)
+
+PLATFORMS
+  arm64-darwin-23
+
+DEPENDENCIES
+  bundler (~> 2.4.22)
+  rake (~> 13.0)
+  scraper-central-ruby!
+
+BUNDLED WITH
+   2.4.22

data/README.md

@@ -0,0 +1,153 @@
+# scraper-central-ruby
+Ruby library to scrape and cache the data
+
+
+## Usage
+
+Add gem `scraper-central-ruby` into Gemfile:
+
+```bash
+gem 'scraper-central-ruby', git: 'git@github.com:patterninc/scraper-central-ruby.git', tag: 'v1.0.0'
+```
+
+```bash
+bundle install
+```
+
+Add below configurations to service where this gem is being used.(ENV variables)
+
+```bash
+SERVER_URL_GET_CACHE=server base url for get cache
+SERVER_URL_PUT_CACHE=server base url for put cache
+```
+
+### New Scraper central object
+
+Use `ScraperCentral.new` to create new instance:
+
+```ruby
+scraper_central = ScraperCentral.new
+```
+
+### Options
+
+Customize the ScraperCentral instance using the provided options:
+
+```ruby
+scraper_central = ScraperCentral.new
+scraper_central.timeout = 45
+scraper_central.tls_verify = false
+scraper_central.enable_js = true
+scraper_central.proxy_name = "ProxyName"
+scraper_central.retry_attr = {
+  count: 5,
+  wait_time: 5,
+  max_wait_time: 5
+}
+```
+
+### Customizing Headers and Query Parameters
+
+To add custom headers and query parameters to your requests:
+
+```ruby
+scraper_central = ScraperCentral.new
+scraper_central.query_params = {
+  "query": "value",
+  "page_wait": "500"
+}
+
+scraper_central.headers = {
+  "X-Custom-Header": "value",
+  "mime-type": "text/html",
+  "Encoding": "gzip",
+  "ContentType": "application/json"
+}
+```
+
+### Cookies
+
+Customizing cookies:
+
+```ruby
+scraper_central = ScraperCentral.new
+scraper_central.cookies = `[
+  {
+    "Name":"amazon-cookie",
+    "Value":"Some session value",
+    "Path":"/",
+    "Domain":"amazon.com",
+    "MaxAge":36000,
+    "HttpOnly":true,
+    "Secure":false
+  },
+  {
+    "Name":"walmart-cookie",
+    "Value":"Some session value",
+    "Path":"/",
+    "Domain":"walmart.com",
+    "MaxAge":72000,
+    "HttpOnly":false,
+    "Secure":true
+  }
+]`
+```
+
+### S3 Key Structure
+
+To generate S3 key we require following methods to set-up:
+
+```ruby
+scraper_central = ScraperCentral.new
+
+scraper_central.s3_key = {
+  country: "US",
+  marketplace: "Amazon",
+  page_type: "detail-page",
+  identifier: "B0BQZBPS4G",
+  page_number: 1
+}
+scraper_central.cache_duration = 360
+
+```
+
+### Fetch Content and Headers
+
+Fetches the URL through the proxy:
+
+```ruby
+scraper_central = ScraperCentral.new
+
+response = scraper_central.fetch("https://example.com")
+
+...
+
+puts "Response: ", response.body
+puts "Status Code: ", response.code
+puts "Headers: ", response.headers
+```
+
+## Documentation
+
+### Configuration Functions
+
+- `scraper_central.proxy_name=`: Sets the proxy service name. e.g. `CrawlBase`, `BrightData`, `ScraperApi`
+- `scraper_central.enable_js=`: Enables or disables JavaScript execution for proxies.
+- `scraper_central.retry_attr=`: Configures retry logic, including the number of attempts, wait time between attempts.
+- `scraper_central.timeout=`: Sets the request timeout in seconds.
+- `scraper_central.tls_verify=`: Configures TLS verification.
+
+### Proxy Methods
+
+- `scraper_central.query_params=`: Sets query parameters to be appended to each request URL.
+- `scraper_central.headers=`: Adds custom headers to requests. (Accept, Accept-Encoding or Content-Type).
+- `scraper_central.cookies=`: Parses a JSON string of cookies and sets them for subsequent requests.
+
+### Proxy Methods For S3 Key
+
+- `scraper_central.s3_key=`: Updates the proxy's target country or S3 key structure country, Marketplace, Pre defined names for page views, e.g. `detail-page`. S3 key value for page identifer e.g. ASIN or Product ID. PageNumber if page is paginated.
+- `scraper_central.cache_duration=`: Age of object stored on S3 bucket.
+
+### Get Content
+
+- `response = fetch(url)`: Makes a configured HTTP request to the specified URL and returns the response object.

data/lib/cache_server.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'uri'
+require 'json'
+require 'logger'
+
+class CacheServer
+  def initialize(proxy_name, enable_js, cache_duration, s3_key)
+    @proxy_name = proxy_name
+    @enable_js = enable_js
+    @cache_duration = cache_duration
+    @s3_key = s3_key
+    @logger = Logger.new($stdout)
+  end
+
+  def get_cache(url)
+    payload = prepare_get_cache_payload(url)
+
+    uri = URI.parse("#{ENV['SERVER_URL_GET_CACHE']}/get-cache")
+    http = Net::HTTP.new(uri.host, uri.port)
+    if uri.scheme == 'https'
+      http.use_ssl = true
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    end
+    request = Net::HTTP::Post.new(uri.request_uri, 'Content-Type' => 'application/json')
+    request.body = payload.to_json
+
+    begin
+      response = http.request(request)
+      if response.content_type.include?('application/json')
+        response_body = JSON.parse(response.body)
+        return '', nil, proxy_from_server(response_body) if response_body.key?('proxyUrl')
+        return response_body['body'], headers_from_server(response_body), nil
+      else
+        @logger.error "Unexpected response type: #{response.content_type}, body: #{response.body}, code: #{response.code}"
+      end
+    rescue StandardError => e
+      @logger.error "Error sending request to server: #{e.message}"
+    end
+    ['', nil, nil]
+  end
+
+  def put_cache(cache_key, page, headers, cookies)
+    payload = {
+      cacheKey: cache_key,
+      page: page,
+      headers: headers,
+      cookies: cookies
+    }
+
+    uri = URI.parse("#{ENV['SERVER_URL_PUT_CACHE']}/put-cache")
+    http = Net::HTTP.new(uri.host, uri.port)
+    if uri.scheme == 'https'
+      http.use_ssl = true
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    end
+
+    request = Net::HTTP::Post.new(uri.request_uri, 'Content-Type' => 'application/json')
+    request.body = payload.to_json
+
+    begin
+      response = http.request(request)
+      if response.code.to_i != 200
+        error_message = "Server returned bad status: #{response.code}"
+        @logger.error error_message
+        raise StandardError, error_message
+      end
+    rescue StandardError => e
+      @logger.error "Error sending cache to server: #{e.message}"
+      raise e
+    end
+  end
+
+  private
+
+  def headers_from_server(response_body)
+    headers = {}
+    if response_body['headers'].is_a?(Hash)
+      response_body['headers'].each do |key, value|
+        headers[key] = value if value.is_a?(String)
+      end
+    end
+    headers
+  end
+
+  def proxy_from_server(response_body)
+    proxy = {}
+    response_body.each do |key, value|
+      proxy[key] = value if value.is_a?(String)
+    end
+    proxy
+  end
+
+  def prepare_get_cache_payload(url)
+    {
+      url: url,
+      proxyName: @proxy_name,
+      country: @s3_key[:country],
+      enableJs: @enable_js,
+      age: @cache_duration,
+      marketplace: @s3_key[:marketplace],
+      pageType: @s3_key[:page_type],
+      identifier: @s3_key[:identifier],
+      pageNumber: @s3_key[:page_number]
+    }
+  end
+end

data/lib/proxy/base.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'uri'
+require 'json'
+require 'openssl'
+require 'response'
+require 'logger'
+
+module Proxy
+  class Base
+    def initialize(params = {})
+      @country = params[:country]
+      @headers = params[:headers] || {}
+      @query_params = params[:query_params] || {}
+      @cookies = params[:cookies] || []
+      @timeout = params[:timeout] || 60
+      @tls_verify = params.fetch(:tls_verify, true)
+      @retry = params[:retry_attr] || {}
+      @enable_js = params.fetch(:enable_js, false)
+      @logger = Logger.new($stdout)
+    end
+
+    def with_retry
+      attempts = 0
+      begin
+        attempts += 1
+        yield
+      rescue StandardError => e
+        raise e unless attempts <= @retry[:count].to_i
+
+        sleep(@retry[:wait_time] || 5)
+        retry
+      end
+    end
+
+    def format_response(response)
+      Response.new(
+        code: response.code.to_i,
+        body: response.body,
+        headers: response.to_hash,
+        cookies: response.get_fields('set-cookie')
+      )
+    end
+
+    def prepare_request(uri, proxy_uri = nil)
+      http = if proxy_uri.nil?
+               Net::HTTP.new(uri.host, uri.port)
+             else
+               Net::HTTP.new(uri.host, uri.port, proxy_uri.host, proxy_uri.port, proxy_uri.user, proxy_uri.password)
+             end
+      if uri.scheme == 'https'
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      end
+
+      http.read_timeout = @timeout
+      http.open_timeout = @timeout
+      http
+    end
+  end
+end

data/lib/proxy/bright_data.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'proxy/base'
+
+module Proxy
+  class BrightData < Proxy::Base
+    def fetch(url, proxy_from_server)
+      uri = URI.parse(url)
+      proxy_uri = URI.parse(proxy_from_server['proxyUrl'])
+      http = prepare_request(uri, proxy_uri)
+
+      uri.query = URI.encode_www_form(@query_params) unless @query_params.empty?
+
+      request = Net::HTTP::Get.new(uri, @headers)
+      @cookies.each do |cookie|
+        request.add_field('Cookie', "#{cookie[:name]}=#{cookie[:value]}")
+      end
+
+      response = with_retry do
+        http.request(request)
+      end
+
+      format_response(response)
+    rescue StandardError => e
+      @logger.error("Request failed error: #{e.message}")
+      nil
+    end
+  end
+end

data/lib/proxy/crawl_base.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require 'proxy/base'
+
+module Proxy
+  class CrawlBase < Proxy::Base
+    def fetch(url, proxy_from_server)
+      uri = URI.parse(proxy_from_server['proxyHost'])
+      uri.query = URI.encode_www_form({
+        'token' => proxy_from_server['proxyToken'],
+        proxy_from_server['proxyCountryKey'] => @country,
+        'url' => url
+      }.merge(@query_params))
+
+      request = Net::HTTP::Get.new(uri, @headers)
+      http = prepare_request(uri)
+
+      @cookies.each do |cookie|
+        request.add_field('Cookie', "#{cookie[:name]}=#{cookie[:value]}")
+      end
+
+      response = with_retry do
+        http.request(request)
+      end
+
+      format_response(response)
+    rescue StandardError => e
+      @logger.error("Request failed error: #{e.message}")
+      nil
+    end
+  end
+end

data/lib/proxy/scraper_api.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require 'proxy/base'
+
+module Proxy
+  class ScraperApi < Proxy::Base
+    def fetch(url, proxy_from_server)
+      uri = URI.parse(proxy_from_server['proxyHost'])
+
+      uri.query = URI.encode_www_form({
+        'api_key' => proxy_from_server['proxyToken'],
+        'render' => @enable_js.to_s,
+        proxy_from_server['proxyCountryKey'] => @country,
+        'url' => url
+      }.merge(@query_params))
+
+      request = Net::HTTP::Get.new(uri, @headers)
+      http = prepare_request(uri)
+
+      @cookies.each do |cookie|
+        request.add_field('Cookie', "#{cookie[:name]}=#{cookie[:value]}")
+      end
+
+      response = with_retry do
+        http.request(request)
+      end
+
+      format_response(response)
+    rescue StandardError => e
+      @logger.error("Request failed error: #{e.message}")
+      nil
+    end
+  end
+end

data/lib/response.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class Response
+  attr_accessor :code, :body, :headers, :cookies
+
+  def initialize(code: 200, body: '', headers: {}, cookies: {})
+    @code = code
+    @body = body
+    @headers = headers
+    @cookies = cookies
+  end
+end

data/lib/scraper_central/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class ScraperCentral
+  VERSION = '1.0.0'
+end

data/lib/scraper_central.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require 'logger'
+require 'cache_server'
+require 'response'
+require 'proxy/bright_data'
+require 'proxy/crawl_base'
+require 'proxy/scraper_api'
+
+class ScraperCentral
+  attr_accessor :cache_duration, :proxy_name, :s3_key, :enable_js, :tls_verify, :headers, :query_params, :cookies,
+                :timeout, :retry_attr
+
+  def initialize
+    @lock = Mutex.new
+    @logger = Logger.new($stdout)
+  end
+
+  def fetch(url)
+    @lock.synchronize do
+      @url = url
+      page_from_server, headers_from_server, proxy_from_server = cache_server.get_cache(@url)
+      if proxy_from_server.nil?
+        print_proxy_values
+        return Response.new(code: 200, body: page_from_server, headers: headers_from_server)
+      else
+        proxy_response = nil
+        params = {
+          country: s3_key[:country],
+          headers: headers,
+          query_params: query_params,
+          cookies: cookies,
+          timeout: timeout,
+          tls_verify: tls_verify,
+          retry_attr: retry_attr,
+          enable_js: enable_js
+        }
+
+        case proxy_from_server['proxyName']
+        when 'BrightData'
+          proxy_response = Proxy::BrightData.new(params).fetch(@url, proxy_from_server)
+        when 'CrawlBase'
+          proxy_response = Proxy::CrawlBase.new(params).fetch(@url, proxy_from_server)
+        when 'ScraperApi'
+          proxy_response = Proxy::ScraperApi.new(params).fetch(@url, proxy_from_server)
+        end
+
+        if proxy_response.nil?
+          @logger.error("Error fetching content from proxy: #{proxy_from_server['proxyName']}")
+          return Response.new(code: 500,
+                              body: StandardError.new("Error fetching content from proxy: #{proxy_from_server['proxyName']}"))
+        end
+
+        Thread.new do
+          cache_server.put_cache(proxy_from_server['cacheKey'], proxy_response.body, proxy_response.headers,
+                                 proxy_response.cookies)
+          @logger.info('Cache successfully sent to server')
+        rescue StandardError => e
+          @logger.error("Error uploading cache to server: #{e.message}")
+        end
+
+        print_proxy_values
+
+        proxy_response
+      end
+    end
+  end
+
+  def print_proxy_values
+    @logger.info("url: #{@url}")
+    unless s3_key.empty?
+      @logger.info("marketplace: #{s3_key[:marketplace]}")
+      @logger.info("country: #{s3_key[:country]}")
+      @logger.info("identifier: #{s3_key[:identifier]}")
+      @logger.info("page_type: #{s3_key[:page_type]}")
+      @logger.info("page_number: #{s3_key[:page_number]}")
+    end
+
+    @logger.info("cache_duration: #{cache_duration}")
+    @logger.info("proxy_name: #{proxy_name}")
+    @logger.info("enable_js: #{enable_js}")
+    @logger.info("tls_verify: #{tls_verify}") if tls_verify
+
+    @logger.info("headers: #{headers}") if headers
+    @logger.info("query_params: #{query_params}") if query_params
+    @logger.info("cookies: #{cookies}") if cookies
+    @logger.info("timeout: #{timeout}") if timeout
+    @logger.info("retry_attr: #{retry_attr}") if retry_attr
+  end
+
+  def cache_server
+    CacheServer.new(proxy_name, enable_js, cache_duration, s3_key)
+  end
+end
+
+# url = 'https://www.amazon.com/ask/questions/asin/B082YK5C1T'
+# scraper_central = ScraperCentral.new()
+# scraper_central.cache_duration = 60
+# scraper_central.proxy_name = 'BrightData'
+# scraper_central.s3_key = {
+#   country: 'us',
+#   marketplace: 'Amazon',
+#   identifier: 'B082YK5C1T',
+#   page_type: 'product-question'
+# }
+# scraper_central.fetch(url)

data/scraper-central-ruby.gemspec

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'scraper_central'
+require 'scraper_central/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'scraper-central-ruby'
+  spec.version       = ScraperCentral::VERSION
+  spec.authors       = ['Patterninc']
+  spec.summary       = 'Scraper central ruby library'
+  spec.email         = ['amol.udage@pattern.com']
+  spec.homepage      = 'https://github.com/patterninc/scraper-central-ruby'
+  spec.license       = 'MIT'
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+  spec.required_ruby_version = '>= 2.7.0'
+
+  # gem install bundler:2.4.22
+  # bundle _2.4.22_
+  spec.add_development_dependency 'bundler', '~> 2.4.22'
+  spec.add_development_dependency 'rake', '~> 13.0'
+  # ... (other development dependencies)
+end

metadata

@@ -0,0 +1,83 @@
+--- !ruby/object:Gem::Specification
+name: scraper-central-ruby
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Patterninc
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2024-07-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.4.22
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.4.22
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+description:
+email:
+- amol.udage@pattern.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- Gemfile.lock
+- README.md
+- lib/cache_server.rb
+- lib/proxy/base.rb
+- lib/proxy/bright_data.rb
+- lib/proxy/crawl_base.rb
+- lib/proxy/scraper_api.rb
+- lib/response.rb
+- lib/scraper_central.rb
+- lib/scraper_central/version.rb
+- scraper-central-ruby.gemspec
+homepage: https://github.com/patterninc/scraper-central-ruby
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.7.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.3
+signing_key:
+specification_version: 4
+summary: Scraper central ruby library
+test_files: []