proxycrawl 0.3.1 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 464bcfcfd4be75be12ec870b5cb2ac5d9f38ed01e1fa2ba44ee1746cf9031795
-  data.tar.gz: 579e140f00efd51ec451b1e33d926d079b3cd5f4353e3a1c001bf37ce04e4e51
+  metadata.gz: c2e422acaf7de6fef3bc43e6241265852a9d699dcf8a114af5aab12b84def901
+  data.tar.gz: a78a2265f2dde7c80918b36167499667d6c85871bb33c2c63e59d727cf9db47e
 SHA512:
-  metadata.gz: 00d314fe67826d76c4ec0470cc5419495d047ed0aa5a408d9d625c45919bd5f1f3c207e0fca30feb66b1b8360e9e7c80f3e9b99967ea6661d5bafb8b981930e1
-  data.tar.gz: 67e1685970d48281b357bcda337fa7bbab40794e9488c5afd3af2810b2cb66a7494e3c8135ca6ff0c085b1d3aa329297b527788d614d2ae578f3b2adb9e5894e
+  metadata.gz: 8b0d9ec78313bdccc5083432f53a1c384280ac2f7725d9309fef20cb488e0e5bc7f625ad03865a34735cd7724d562a45cf91c1abd86e11f7dc610c36aabb354f
+  data.tar.gz: 95a92e684149d66a072f47c98d41f4de23cd538d0a85685ec17ff979e3c773e6ca8061191d3455411f6a6dddefe272b6c291cc09e117da2e2d367fca60311fec

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2021 ProxyCrawl
+Copyright (c) 2023 ProxyCrawl
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -1,3 +1,9 @@
+# DEPRECATION NOTICE
+
+> :warning: **IMPORTANT:** This gem is no longer maintained or supported. For the latest updates, please use our new gem at [crawlbase-ruby](https://github.com/crawlbase-source/crawlbase-ruby).
+
+---
+
 # ProxyCrawl
 
 Dependency free gem for scraping and crawling websites using the ProxyCrawl API.
@@ -361,4 +367,4 @@ Everyone interacting in the Proxycrawl project’s codebases, issue trackers, ch
 
 ---
 
-Copyright 2021 ProxyCrawl
+Copyright 2023 ProxyCrawl

data/lib/proxycrawl/api.rb

@@ -6,7 +6,7 @@ require 'uri'
 
 module ProxyCrawl
   class API
-    attr_reader :token, :body, :status_code, :original_status, :pc_status, :url, :storage_url
+    attr_reader :token, :body, :timeout, :status_code, :original_status, :pc_status, :url, :storage_url
 
     INVALID_TOKEN = 'Token is required'
     INVALID_URL = 'URL is required'
@@ -15,14 +15,22 @@ module ProxyCrawl
       raise INVALID_TOKEN if options[:token].nil?
 
       @token = options[:token]
+      @timeout = options[:timeout] || 120
     end
 
     def get(url, options = {})
       raise INVALID_URL if url.empty?
 
       uri = prepare_uri(url, options)
+      req = Net::HTTP::Get.new(uri)
 
-      response = Net::HTTP.get_response(uri)
+      req_options = {
+        read_timeout: timeout,
+        use_ssl: uri.scheme == 'https',
+        verify_mode: OpenSSL::SSL::VERIFY_NONE
+      }
+
+      response = Net::HTTP.start(uri.hostname, uri.port, req_options) { |http| http.request(req) }
 
       prepare_response(response, options[:format])
 

data/lib/proxycrawl/leads_api.rb

@@ -6,7 +6,7 @@ require 'uri'
 
 module ProxyCrawl
   class LeadsAPI
-    attr_reader :token, :body, :status_code, :success, :remaining_requests
+    attr_reader :token, :timeout, :body, :status_code, :success, :remaining_requests
 
     INVALID_TOKEN = 'Token is required'
     INVALID_DOMAIN = 'Domain is required'
@@ -15,6 +15,7 @@ module ProxyCrawl
       raise INVALID_TOKEN if options[:token].nil? || options[:token].empty?
 
       @token = options[:token]
+      @timeout = options[:timeout] || 120
     end
 
     def get(domain)
@@ -23,7 +24,15 @@ module ProxyCrawl
       uri = URI('https://api.proxycrawl.com/leads')
       uri.query = URI.encode_www_form({ token: token, domain: domain })
 
-      response = Net::HTTP.get_response(uri)
+      req = Net::HTTP::Get.new(uri)
+
+      req_options = {
+        read_timeout: timeout,
+        use_ssl: uri.scheme == 'https',
+        verify_mode: OpenSSL::SSL::VERIFY_NONE
+      }
+
+      response = Net::HTTP.start(uri.hostname, uri.port, req_options) { |http| http.request(req) }
       @status_code = response.code.to_i
       @body = response.body
 

data/lib/proxycrawl/storage_api.rb

@@ -6,7 +6,7 @@ require 'uri'
 
 module ProxyCrawl
   class StorageAPI
-    attr_reader :token, :original_status, :pc_status, :url, :status_code, :rid, :body, :stored_at
+    attr_reader :token, :timeout, :original_status, :pc_status, :url, :status_code, :rid, :body, :stored_at
 
     INVALID_TOKEN = 'Token is required'
     INVALID_RID = 'RID is required'
@@ -18,6 +18,7 @@ module ProxyCrawl
       raise INVALID_TOKEN if options[:token].nil? || options[:token].empty?
 
       @token = options[:token]
+      @timeout = options[:timeout] || 120
     end
 
     def get(url_or_rid, format = 'html')
@@ -25,7 +26,16 @@ module ProxyCrawl
 
       uri = URI(BASE_URL)
       uri.query = URI.encode_www_form({ token: token, format: format }.merge(decide_url_or_rid(url_or_rid)))
-      response = Net::HTTP.get_response(uri)
+
+      req = Net::HTTP::Get.new(uri)
+
+      req_options = {
+        read_timeout: timeout,
+        use_ssl: uri.scheme == 'https',
+        verify_mode: OpenSSL::SSL::VERIFY_NONE
+      }
+
+      response = Net::HTTP.start(uri.hostname, uri.port, req_options) { |http| http.request(req) }
 
       res = format == 'json' ? JSON.parse(response.body) : response
 

data/lib/proxycrawl/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ProxyCrawl
-  VERSION = '0.3.1'
+  VERSION = '1.0.1'
 end

data/proxycrawl.gemspec

@@ -28,4 +28,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "webmock", "~> 3.4"
   spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "rake", "~> 12.3.3"
+
+  # Deprecation warning
+  spec.post_install_message = "DEPRECATION WARNING: This package is no longer maintained due to rebranding. Please use the 'crawlbase' gem instead. More info: https://github.com/crawlbase-source/crawlbase-ruby"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: proxycrawl
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 1.0.1
 platform: ruby
 authors:
 - proxycrawl
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-07-14 00:00:00.000000000 Z
+date: 2023-07-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -94,7 +94,8 @@ homepage: https://github.com/proxycrawl/proxycrawl-ruby
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message: 'DEPRECATION WARNING: This package is no longer maintained due
+  to rebranding. Please use the ''crawlbase'' gem instead. More info: https://github.com/crawlbase-source/crawlbase-ruby'
 rdoc_options: []
 require_paths:
 - lib
@@ -109,7 +110,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: ProxyCrawl API client for web scraping and crawling