httply 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f4f94c9f550d3248f2461513736182b62fea724bf6afbd183d84576d2f1e8db8
-  data.tar.gz: 7dc91382584aa43a6b9c4ec2291b029c17d267bac38d6705df03ec79c3c236d1
+  metadata.gz: 1a9a2b6ebd2873064c17ff349e01c280258cf3672aa3efa9fc4d9552d4d64852
+  data.tar.gz: 304af45cd9cf5870fb14e1883d44fb8306cd01cb28b397f456a35d52736767f6
 SHA512:
-  metadata.gz: 78d8610e593ecf6fc2068b17f779ed6895c8eccc8f289f4ccf1967c44aea1b3e240d09417d863eb95b7a512eb7a46f495c362ecef2b00d7800244a4a78e743eb
-  data.tar.gz: 4190a58a8f1c10a6a09635eb9d8478b6762083e7ee1a27e8b4ac4634d586296bfdce67f1ad7547e2aab9aabadad9ed107375cfffebf9182885359cd7aaa6e473
+  metadata.gz: 2276fbddf8538d72109e60498cdac15f6e3ce0a595b1a37a126b19965d2e1b244965c8a86df0f2c78196c68a267dc3d14dbebfac1bf01e6e91935e19003a8bdd
+  data.tar.gz: bdfad9a05aa7b634193faf62f3768e80414591719bf0a7a06f9e8449378ca4d378e9684146672ae9550f58ba022fc9294148ede57570989f917065029fc478a2

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    httply (0.1.1)
+    httply (0.1.4)
       agents (>= 0.1.4)
       faraday (>= 0.15.4)
       faraday_middleware (>= 0.13.1)

data/README.md

@@ -2,7 +2,7 @@
 
 Httply is a lightweight wrapper around Faraday to support automatic randomization of proxies and user agents, amongst other things.
 
-Randomized proxy switching support is provided by the [proxied](https://github.com/SebastianJ/proxied) gem and randomized user agent is provided by the [agents](https://github.com/SebastianJ/agents) gem.
+Randomized proxy switching support is provided by the [proxied](https://github.com/SebastianJ/proxied) gem and randomized user agent support is provided by the [agents](https://github.com/SebastianJ/agents) gem.
 
 Randomized/automatic proxy switching is currently only supported in conjunction with the [proxied](https://github.com/SebastianJ/proxied) gem. Manual proxy support is obviously supported irregardless of using [proxied](https://github.com/SebastianJ/proxied) or not.
 
@@ -26,16 +26,25 @@ Or install it yourself as:
 
 ### Instance
 
-Httply can either be used as an instance, e.g:
+Httply can be used as an instance:
 
 ```ruby
 client    =   Httply::Client.new(host: "https://www.google.com")
 response  =   client.get("/webhp", parameters: {hl: :en})
 ```
 
+Httply can also memoize the Faraday connection (e.g. if you're invoking an API and using the same proxy and user agent for every request):
+
+```ruby
+client    =   Httply::Client.new(host: "https://www.google.com", memoize: true)
+response  =   client.get("/webhp", parameters: {hl: :en})
+```
+
+The difference is that for the first example user-agent and proxy will automatically be rotated (if not providing a specifc user-agent or proxy) on every request whereas in the second example the user-agent and proxy will be kept for every request.
+
 ### Class method
 
-Or directly as a class method:
+Httply can also be invoked on a class level:
 
 ```ruby
 response  =   Httply.get("https://www.google.com/webhp", parameters: {hl: :en})
@@ -45,10 +54,10 @@ response  =   Httply.get("https://www.google.com/webhp", parameters: {hl: :en})
 
 HTML parsing requires that Nokogiri has been required elsewhere and is accessible for Httply.
 
-You can also force parsing responses as HTML using Nokogiri:
+If Nokogiri is available "text/html" responses will automatically be parsed by Nokogiri.
 
 ```ruby
-response  =   Httply.get("https://www.google.com/webhp", parameters: {hl: :en}, as: :html)
+response  =   Httply.get("https://www.google.com/webhp", parameters: {hl: :en})
 ```
 
 response.body will now return a Nokogiri::HTML::Document

data/lib/httply/client.rb

@@ -1,63 +1,64 @@
 module Httply
   class Client
     attr_accessor :host, :configuration
+    attr_accessor :memoize, :connection
     
     include ::Httply::Proxies
     
-    def initialize(host: nil, configuration: ::Httply.configuration)
-      self.host             =   host
+    def initialize(host: nil, configuration: ::Httply.configuration, memoize: false)
+      self.host             =   ::Httply::Utilities::Uri.correct_host(host)
       self.configuration    =   configuration
+      self.memoize          =   memoize
+      self.connection       =   nil
     end
     
-    def to_uri(path)
-      path                  =   path.gsub(/^\//i, "")
-      
-      if path !~ /^http(s)?:\/\// && !self.host.to_s.empty?
-        host_part           =   self.host =~ /^http(s)?:\/\// ? self.host : "https://#{self.host}"
-        path                =   "#{host_part}/#{path}"
-      end
-      
-      return path
+    def setup(host: nil, headers: {}, options: {})
+      self.connection     ||=   configure(host: host, headers: headers, options: options)
     end
     
-    def get(path, parameters: {}, headers: {}, options: {}, as: nil)
-      request path, method: :get, parameters: parameters, headers: headers, options: options, as: as
+    def get(path, parameters: {}, headers: {}, options: {})
+      request path, method: :get, parameters: parameters, headers: headers, options: options
     end
     
-    def head(path, parameters: {}, headers: {}, options: {}, as: nil)
-      request path, method: :head, parameters: parameters, headers: headers, options: options, as: as
+    def head(path, parameters: {}, headers: {}, options: {})
+      request path, method: :head, parameters: parameters, headers: headers, options: options
     end
 
-    def post(path, parameters: {}, data: {}, headers: {}, options: {}, as: nil)
-      request path, method: :post, parameters: parameters, data: data, headers: headers, options: options, as: as
+    def post(path, parameters: {}, data: {}, headers: {}, options: {})
+      request path, method: :post, parameters: parameters, data: data, headers: headers, options: options
     end
     
-    def put(path, parameters: {}, data: {}, headers: {}, options: {}, as: nil)
-      request path, method: :put, parameters: parameters, data: data, headers: headers, options: options, as: as
+    def put(path, parameters: {}, data: {}, headers: {}, options: {})
+      request path, method: :put, parameters: parameters, data: data, headers: headers, options: options
     end
     
-    def patch(path, parameters: {}, data: {}, headers: {}, options: {}, as: nil)
-      request path, method: :patch, parameters: parameters, data: data, headers: headers, options: options, as: as
+    def patch(path, parameters: {}, data: {}, headers: {}, options: {})
+      request path, method: :patch, parameters: parameters, data: data, headers: headers, options: options
     end
     
-    def delete(path, parameters: {}, data: {}, headers: {}, options: {}, as: nil)
-      request path, method: :delete, parameters: parameters, data: data, headers: headers, options: options, as: as
+    def delete(path, parameters: {}, data: {}, headers: {}, options: {})
+      request path, method: :delete, parameters: parameters, data: data, headers: headers, options: options
     end
     
-    def request(path, method: :get, parameters: {}, data: {}, headers: {}, options: {}, as: nil)
-      connection                =   setup(path, method: method, headers: headers, options: options, as: as)
+    def request(path, method: :get, parameters: {}, data: {}, headers: {}, options: {})
+      host                      =   !self.host.to_s.empty? ? self.host : ::Httply::Utilities::Uri.parse_host(path)
+      path                      =   ::Httply::Utilities::Uri.to_path(path)
+      connection                =   self.memoize ? setup(host: host, headers: headers, options: options) : configure(host: host, headers: headers, options: options)
   
       response                  =   case method
         when :get
           connection.get do |request|
+            request.url path
             request.params      =   parameters if parameters && !parameters.empty?
           end
         when :head
           connection.head do |request|
+            request.url path
             request.params      =   parameters if parameters && !parameters.empty?
           end
         when :post, :put, :patch, :delete
           connection.send(method) do |request|
+            request.url path
             request.body        =   data if data && !data.empty?
             request.params      =   parameters if parameters && !parameters.empty?
           end
@@ -66,35 +67,35 @@ module Httply
       return response
     end
     
-    def setup(path, method: :get, headers: {}, options: {}, as: nil)
+    def configure(host:, headers: {}, options: {})
       client_options            =   options.fetch(:client, {})
-      follow_redirects          =   options.fetch(:follow_redirects, true)
-      redirect_limit            =   options.fetch(:redirects_limit, 10)
-      proxy                     =   determine_proxy(options.fetch(:proxy, nil))
       
-      url                       =   to_uri(path)
+      request_options           =   options.fetch(:request, {})
+      redirects                 =   request_options.fetch(:redirects, 10)
       
-      headers                   =   {"User-Agent" => ::Agents.random_user_agent(options.fetch(:user_agent_device, :desktop))}.merge(headers)
+      proxy                     =   determine_proxy(options.fetch(:proxy, nil))
       
-      connection                =   ::Faraday.new(url, client_options) do |builder|
+      headers["User-Agent"]     =   headers.fetch("User-Agent", ::Agents.random_user_agent(options.fetch(:user_agent_device, :desktop)))
+      
+      connection                =   ::Faraday.new(host, client_options) do |builder|
         builder.options[:timeout]         =   options.fetch(:timeout, nil)      if options.fetch(:timeout, nil)
         builder.options[:open_timeout]    =   options.fetch(:open_timeout, nil) if options.fetch(:open_timeout, nil)
         
         builder.headers         =   headers
         
-        builder.request :url_encoded if [:post, :put, :patch, :delete].include?(method)
-        
-        builder.response :logger if self.configuration.verbose
+        builder.request  :url_encoded if request_options.fetch(:url_encoded, false)
+        builder.request  :json        if request_options.fetch(:json, false)
         
-        builder.response :xml,  content_type: /\bxml$/  if as.eql?(:xml)
-        builder.response :json, content_type: /\bjson$/ if as.eql?(:json)
-        builder.use ::Httply::Middlewares::ParseHtml      if as.eql?(:html)
+        builder.response :logger      if self.configuration.verbose
+        builder.response :xml,  content_type: /\bxml$/
+        builder.response :json, content_type: /\bjson$/
+        builder.use ::Httply::Middlewares::ParseHtml, content_type: /\btext\/html$/
         
-        builder.use ::FaradayMiddleware::FollowRedirects, limit: redirect_limit if follow_redirects && redirect_limit && redirect_limit > 0
+        builder.use ::FaradayMiddleware::FollowRedirects, limit: redirects if redirects && redirects > 0
         
         if proxy
           builder.proxy         =   generate_faraday_proxy(proxy)
-          puts "[Httply::Client] - Will use proxy: #{builder.proxy.inspect}" if self.configuration.verbose
+          log("Will use proxy: #{builder.proxy.inspect}")
         end
     
         builder.adapter self.configuration.faraday.fetch(:adapter, ::Faraday.default_adapter)
@@ -103,5 +104,9 @@ module Httply
       return connection
     end
     
+    def log(message)
+      puts "[Httply::Client] - #{message}" if self.configuration.verbose
+    end
+    
   end
 end

data/lib/httply/utilities/uri.rb

@@ -0,0 +1,33 @@
+module Httply
+  module Utilities
+    class Uri
+      class << self
+        
+        def correct_host(host)
+          if !host.to_s.empty?
+            host                    =   host =~ /^http(s)?:\/\//i ? host : "https://#{host}"
+          end
+      
+          return host
+        end
+    
+        def parse_host(url)
+          host                      =   nil
+      
+          if host.to_s.empty? && url =~ /^http(s)?:\/\//
+            uri                     =   URI(url)
+            host                    =   "#{uri.scheme}://#{uri.host}"
+          end
+      
+          return host
+        end
+    
+        def to_path(path)
+          path                      =   path =~ /^http(s)?:\/\// ? URI(path).path : path
+          path                      =   path =~ /^\// ? path : "/#{path}"
+        end
+        
+      end
+    end
+  end
+end

data/lib/httply/version.rb

@@ -1,3 +1,3 @@
 module Httply
-  VERSION = "0.1.4"
+  VERSION = "0.1.5"
 end

data/lib/httply.rb

@@ -2,11 +2,15 @@ require "faraday"
 require "faraday_middleware"
 require "agents"
 
+require "uri"
+
 require "httply/version"
 require "httply/configuration"
 
 require "httply/middlewares/html"
 
+require "httply/utilities/uri"
+
 require "httply/proxies"
 require "httply/client"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: httply
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - Sebastian Johnsson
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-05-22 00:00:00.000000000 Z
+date: 2019-06-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday
@@ -162,6 +162,7 @@ files:
 - lib/httply/configuration.rb
 - lib/httply/middlewares/html.rb
 - lib/httply/proxies.rb
+- lib/httply/utilities/uri.rb
 - lib/httply/version.rb
 homepage: https://github.com/SebastianJ/httply
 licenses: