proxycrawl 0.3.1 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 464bcfcfd4be75be12ec870b5cb2ac5d9f38ed01e1fa2ba44ee1746cf9031795
-  data.tar.gz: 579e140f00efd51ec451b1e33d926d079b3cd5f4353e3a1c001bf37ce04e4e51
+  metadata.gz: c225955f21200efc0790c401385727e3b9dd9bc20f785ce768f44f687f55105b
+  data.tar.gz: 0a38e44493065a9779366d4ab5372771bf92b020da513059c0cf20b69e2d696a
 SHA512:
-  metadata.gz: 00d314fe67826d76c4ec0470cc5419495d047ed0aa5a408d9d625c45919bd5f1f3c207e0fca30feb66b1b8360e9e7c80f3e9b99967ea6661d5bafb8b981930e1
-  data.tar.gz: 67e1685970d48281b357bcda337fa7bbab40794e9488c5afd3af2810b2cb66a7494e3c8135ca6ff0c085b1d3aa329297b527788d614d2ae578f3b2adb9e5894e
+  metadata.gz: 2d81900546063ff3ddbbe660f528b8cbc04336916bce468f37c8791370b7dd99c36d32915dd471ff886bc6fb8fb498ddccc4adfb40c95009849e25a1f4938805
+  data.tar.gz: cd167890621d7d8bcbdf1c2502e545c9ea40c95abf437927e443dc1e3694d8393fae4d1c521e7b34227c4a962e172cb085697169d43f9ee1ff99369649e6c36f

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2021 ProxyCrawl
+Copyright (c) 2022 ProxyCrawl
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -361,4 +361,4 @@ Everyone interacting in the Proxycrawl project’s codebases, issue trackers, ch
 
 ---
 
-Copyright 2021 ProxyCrawl
+Copyright 2022 ProxyCrawl

data/lib/proxycrawl/api.rb

@@ -6,7 +6,7 @@ require 'uri'
 
 module ProxyCrawl
   class API
-    attr_reader :token, :body, :status_code, :original_status, :pc_status, :url, :storage_url
+    attr_reader :token, :body, :timeout, :status_code, :original_status, :pc_status, :url, :storage_url
 
     INVALID_TOKEN = 'Token is required'
     INVALID_URL = 'URL is required'
@@ -15,14 +15,22 @@ module ProxyCrawl
       raise INVALID_TOKEN if options[:token].nil?
 
       @token = options[:token]
+      @timeout = options[:timeout] || 120
     end
 
     def get(url, options = {})
       raise INVALID_URL if url.empty?
 
       uri = prepare_uri(url, options)
+      req = Net::HTTP::Get.new(uri)
 
-      response = Net::HTTP.get_response(uri)
+      req_options = {
+        read_timeout: timeout,
+        use_ssl: uri.scheme == 'https',
+        verify_mode: OpenSSL::SSL::VERIFY_NONE
+      }
+
+      response = Net::HTTP.start(uri.hostname, uri.port, req_options) { |http| http.request(req) }
 
       prepare_response(response, options[:format])
 

data/lib/proxycrawl/leads_api.rb

@@ -6,7 +6,7 @@ require 'uri'
 
 module ProxyCrawl
   class LeadsAPI
-    attr_reader :token, :body, :status_code, :success, :remaining_requests
+    attr_reader :token, :timeout, :body, :status_code, :success, :remaining_requests
 
     INVALID_TOKEN = 'Token is required'
     INVALID_DOMAIN = 'Domain is required'
@@ -15,6 +15,7 @@ module ProxyCrawl
       raise INVALID_TOKEN if options[:token].nil? || options[:token].empty?
 
       @token = options[:token]
+      @timeout = options[:timeout] || 120
     end
 
     def get(domain)
@@ -23,7 +24,15 @@ module ProxyCrawl
       uri = URI('https://api.proxycrawl.com/leads')
       uri.query = URI.encode_www_form({ token: token, domain: domain })
 
-      response = Net::HTTP.get_response(uri)
+      req = Net::HTTP::Get.new(uri)
+
+      req_options = {
+        read_timeout: timeout,
+        use_ssl: uri.scheme == 'https',
+        verify_mode: OpenSSL::SSL::VERIFY_NONE
+      }
+
+      response = Net::HTTP.start(uri.hostname, uri.port, req_options) { |http| http.request(req) }
       @status_code = response.code.to_i
       @body = response.body
 

data/lib/proxycrawl/storage_api.rb

@@ -6,7 +6,7 @@ require 'uri'
 
 module ProxyCrawl
   class StorageAPI
-    attr_reader :token, :original_status, :pc_status, :url, :status_code, :rid, :body, :stored_at
+    attr_reader :token, :timeout, :original_status, :pc_status, :url, :status_code, :rid, :body, :stored_at
 
     INVALID_TOKEN = 'Token is required'
     INVALID_RID = 'RID is required'
@@ -18,6 +18,7 @@ module ProxyCrawl
       raise INVALID_TOKEN if options[:token].nil? || options[:token].empty?
 
       @token = options[:token]
+      @timeout = options[:timeout] || 120
     end
 
     def get(url_or_rid, format = 'html')
@@ -25,7 +26,16 @@ module ProxyCrawl
 
       uri = URI(BASE_URL)
       uri.query = URI.encode_www_form({ token: token, format: format }.merge(decide_url_or_rid(url_or_rid)))
-      response = Net::HTTP.get_response(uri)
+
+      req = Net::HTTP::Get.new(uri)
+
+      req_options = {
+        read_timeout: timeout,
+        use_ssl: uri.scheme == 'https',
+        verify_mode: OpenSSL::SSL::VERIFY_NONE
+      }
+
+      response = Net::HTTP.start(uri.hostname, uri.port, req_options) { |http| http.request(req) }
 
       res = format == 'json' ? JSON.parse(response.body) : response
 

data/lib/proxycrawl/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ProxyCrawl
-  VERSION = '0.3.1'
+  VERSION = '1.0.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: proxycrawl
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 1.0.0
 platform: ruby
 authors:
 - proxycrawl
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-07-14 00:00:00.000000000 Z
+date: 2022-08-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -109,7 +109,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: ProxyCrawl API client for web scraping and crawling