scraper-central-ruby 1.0.0 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3a80d1ba8b02b21d51a048f7b153f04e23176ffcd5fc258f004ac903776fe831
-  data.tar.gz: 9982839e9a725452a3f26022ccbd46eabea0aa8ac3b94fd1e4bdcda172ae58cc
+  metadata.gz: 812034013aa0b9d1fbec3961dad038a59b3c01c26e5738b5da8ee31d0af8acd1
+  data.tar.gz: 2da443589638a347f9ee81a73141b43d4073bb9dd148371421b9843906886e36
 SHA512:
-  metadata.gz: 20f9efb238092af60d758971d8a84752a1cf24b9f927795352be2d66fc47784b3050b629300e0c7ed9b8baa04521970600851ce44d3a2b592f556a489a35ae5f
-  data.tar.gz: c464c0c5a20b58366f2e738aee1cabb1b66555c3d9217b277fe9d9ab3433968f596be12dad4717738017b725b9044e3db0e24797aeec146d93d3abe1527119ce
+  metadata.gz: 8070123f403672e8b2f32fa6f26297447eb1f3b0bf8fcef4452b52491f82bf27fdb3e0f40a0f5e411c88372653335a94c4ed075bffa4ee6bdde589bcd1a3a4a1
+  data.tar.gz: 1112f3744c8c62db525394d4091e786640126a1b4a6bcaa1efa65f1ea850d5213ffcd0538df513197901d70c7f4167bf16821e1fa956e3e18be3d7bae50828ed

data/Gemfile.lock

@@ -1,11 +1,13 @@
 PATH
   remote: .
   specs:
-    scraper-central-ruby (1.0.0)
+    scraper-central-ruby (1.0.2)
+      brotli (~> 0.5.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    brotli (0.5.0)
     rake (13.2.1)
 
 PLATFORMS

data/README.md

@@ -117,6 +117,18 @@ Fetches the URL through the proxy:
 
 ```ruby
 scraper_central = ScraperCentral.new
+scraper_central.proxy_name = "BrightData"
+scraper_central.cache_duration = 360
+scraper_central.retry_attr = {
+  count: 5
+}
+scraper_central.s3_key = {
+  country: "US",
+  marketplace: "Amazon",
+  page_type: "product-question",
+  identifier: "B0BQZBPS4G",
+  page_number: 1
+}
 
 response = scraper_central.fetch("https://example.com")
 

data/lib/proxy/base.rb

@@ -6,6 +6,7 @@ require 'json'
 require 'openssl'
 require 'response'
 require 'logger'
+require 'brotli'
 
 module Proxy
   class Base
@@ -35,14 +36,33 @@ module Proxy
     end
 
     def format_response(response)
+      if response.kind_of?(Net::HTTPSuccess) && response.header['Content-Encoding'].eql?('gzip')
+        sio = StringIO.new(response.body)
+        gz = Zlib::GzipReader.new(sio)
+        page_content = gz.read()
+      elsif response.kind_of?(Net::HTTPSuccess) && response.header['Content-Encoding'].eql?('br')
+        # Decode response which is in brotli encryption and save it as html
+        page_content = Brotli.inflate(response.body)
+      else
+        page_content = response.body
+      end
+
       Response.new(
         code: response.code.to_i,
-        body: response.body,
-        headers: response.to_hash,
+        body: page_content,
+        headers: get_headers(response),
         cookies: response.get_fields('set-cookie')
       )
     end
 
+    def get_headers(response)
+      headers = {}
+      response.each_header do |key, value|
+        headers[key] = value
+      end
+      headers
+    end
+
     def prepare_request(uri, proxy_uri = nil)
       http = if proxy_uri.nil?
                Net::HTTP.new(uri.host, uri.port)

data/lib/scraper_central/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class ScraperCentral
-  VERSION = '1.0.0'
+  VERSION = '1.0.2'
 end

data/scraper-central-ruby.gemspec

@@ -14,6 +14,10 @@ Gem::Specification.new do |spec|
   spec.homepage      = 'https://github.com/patterninc/scraper-central-ruby'
   spec.license       = 'MIT'
 
+
+  # Add the brotli gem dependency here
+  spec.add_dependency 'brotli', '~> 0.5.0'
+
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files         = Dir.chdir(File.expand_path(__dir__)) do

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: scraper-central-ruby
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.2
 platform: ruby
 authors:
 - Patterninc
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-07-11 00:00:00.000000000 Z
+date: 2024-07-19 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: brotli
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement