benchmark-http 0.9.0 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8b3803e60e468cb173cbaf5df1556f95fc7dfe89531fb785530050875a5a32e1
-  data.tar.gz: a5c54afd2afb70743f6d589eaa784c593a5f78d96e59c87c996ef90a7aa968fb
+  metadata.gz: '069082c04439ca3f6d2a49daa213721646b46051c0ea6b9c55d0247adfdcbeaf'
+  data.tar.gz: 041fc41a0379bac2264ff90dde3dbc03ef6a20216deeb75fa3fdb818294bdf7f
 SHA512:
-  metadata.gz: aff57e060c1239d34ef229f33c6141beefdea04970adf8608d119824cc512bcb5e994df5df3290023f03e1f3a3563184a8437b377ad01a26d1576718cfa45715
-  data.tar.gz: 78d67c5fd5dfdbe15306723711ea4b274278e9c694d287105f3b808d725b10ae500c64007f67497eddd655d4b8a8603c02395b99b1db23422fd725711e649209
+  metadata.gz: 8ff5661cfa6ee80bbd0737b87a872da7e180bdb27536dbcc321625e331c8557333e314a1eedc9f1af1e9924cb556146540463aed89242b186c336032bd892289
+  data.tar.gz: b842c305fc9ab7a18bb1c0f7dbcc2e6b7933dbae5cdfed237b040fa2c1a79edc16cfdc74083213067feca4493cda4a16d4f670e654d668e3870335f2090776ce

data/.travis.yml

@@ -7,6 +7,7 @@ matrix:
     - rvm: 2.4
     - rvm: 2.5
     - rvm: 2.6
+    - rvm: 2.7    
     - rvm: jruby-head
       env: JRUBY_OPTS="--debug -X+O"
     - rvm: ruby-head

data/lib/benchmark/http/command/spider.rb

@@ -18,13 +18,8 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
-require_relative '../seconds'
-require_relative '../statistics'
-require_relative '../links_filter'
+require_relative '../spider'
 
-require 'async'
-require 'async/http/client'
-require 'async/http/endpoint'
 require 'async/await'
 
 require 'samovar'
@@ -46,97 +41,19 @@ module Benchmark
 				many :urls, "One or more hosts to benchmark"
 				
 				def log(method, url, response)
-					puts "#{method} #{url} -> #{response.version} #{response.status} (#{response.body&.length || 'unspecified'} bytes)"
-					
-					response.headers.each do |key, value|
-						puts "\t#{key}: #{value}"
-					end if @options[:headers]
-				end
-				
-				def extract_links(url, response)
-					base = url
-					
-					body = response.read
-					
-					begin
-						filter = LinksFilter.parse(body)
-					rescue
-						Async.logger.error($!)
-						return []
-					end
-					
-					if filter.base
-						base = base + filter.base
-					end
-					
-					filter.links.collect do |href|
-						next if href.nil? or href.empty?
+					Async.logger.call(self, severity: (response.failure? ? :warn : :info)) do |buffer|
+						buffer.puts "#{method} #{url} -> #{response.version} #{response.status} (#{response.body&.length || 'unspecified'} bytes)"
 						
-						begin
-							full_url = base + href
-							
-							if full_url.host == url.host && full_url.kind_of?(URI::HTTP)
-								yield full_url
-							end
-						rescue ArgumentError, URI::InvalidURIError
-							puts "Could not fetch #{href}, relative to #{base}."
-						end
-					end.compact
-				end
-				
-				async def fetch(statistics, client, url, depth = @options[:depth], fetched = Set.new)
-					return if fetched.include?(url) or depth == 0
-					
-					fetched << url
-					
-					request_uri = url.request_uri
-					
-					response = client.head(request_uri).tap(&:read)
-					
-					log("HEAD", url, response)
-					
-					if response.redirection?
-						location = url + response.headers['location']
-						if location.host == url.host
-							puts "Following redirect to #{location}..."
-							return fetch(statistics, client, location, depth-1, fetched).wait
-						else
-							puts "Ignoring redirect to #{location}."
-							return
-						end
-					end
-					
-					content_type = response.headers['content-type']
-					unless content_type&.start_with? 'text/html'
-						puts "Unsupported content type: #{content_type}"
-						return
+						response.headers.each do |key, value|
+							buffer.puts "\t#{key}: #{value}"
+						end if @options[:headers]
 					end
-					
-					response = statistics.measure do
-						client.get(request_uri)
-					end
-					
-					log("GET", url, response)
-					
-					extract_links(url, response) do |href|
-						fetch(statistics, client, href, depth - 1, fetched)
-					end.each(&:wait)
-				rescue Async::TimeoutError
-					Async.logger.error("Timeout while fetching #{url}")
-				rescue StandardError
-					Async.logger.error($!)
 				end
 				
-				async def call
-					statistics = Statistics.new
+				sync def call
+					spider = HTTP::Spider.new(depth: @options[:depth])
 					
-					@urls.each do |url|
-						endpoint = Async::HTTP::Endpoint.parse(url, timeout: 10)
-						
-						Async::HTTP::Client.open(endpoint, endpoint.protocol, connection_limit: 4) do |client|
-							fetch(statistics, client, endpoint.url).wait
-						end
-					end
+					statistics = spider.call(@urls, &self.method(:log))
 					
 					statistics.print
 					

data/lib/benchmark/http/spider.rb

@@ -0,0 +1,132 @@
+# Copyright, 2018, by Samuel G. D. Williams. <http://www.codeotaku.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require_relative 'seconds'
+require_relative 'statistics'
+require_relative 'links_filter'
+
+require 'async'
+require 'async/http/client'
+require 'async/http/endpoint'
+require 'async/await'
+
+require 'uri'
+
+module Benchmark
+	module HTTP
+		class Spider
+			include Async::Await
+			
+			def initialize(depth: nil)
+				@depth = depth
+			end
+			
+			def extract_links(url, response)
+				base = url
+				
+				body = response.read
+				
+				begin
+					filter = LinksFilter.parse(body)
+				rescue
+					Async.logger.error(self) {$!}
+					return []
+				end
+				
+				if filter.base
+					base = base + filter.base
+				end
+				
+				filter.links.collect do |href|
+					next if href.nil? or href.empty?
+					
+					begin
+						full_url = base + href
+						
+						if full_url.host == url.host && full_url.kind_of?(URI::HTTP)
+							yield full_url
+						end
+					rescue ArgumentError, URI::InvalidURIError
+						Async.logger.warn(self) {"Could not fetch #{href}, relative to #{base}!"}
+						next # Don't accumulate an item into the resulting array.
+					end
+				end.compact
+			end
+			
+			async def fetch(statistics, client, url, depth = @depth, fetched = Set.new, &block)
+				return if fetched.include?(url) or depth == 0
+				
+				fetched << url
+				
+				request_uri = url.request_uri
+				
+				response = client.head(request_uri).tap(&:read)
+				
+				yield("HEAD", url, response) if block_given?
+				
+				if response.redirection?
+					location = url + response.headers['location']
+					if location.host == url.host
+						Async.logger.info(self) {"Following redirect to #{location}..."}
+						fetch(statistics, client, location, depth-1, fetched, &block).wait
+						return
+					else
+						Async.logger.info(self) {"Ignoring redirect to #{location}."}
+						return
+					end
+				end
+				
+				content_type = response.headers['content-type']
+				unless content_type&.start_with? 'text/html'
+					# puts "Unsupported content type: #{content_type}"
+					return
+				end
+				
+				response = statistics.measure do
+					client.get(request_uri)
+				end
+				
+				yield("GET", url, response) if block_given?
+				
+				extract_links(url, response) do |href|
+					fetch(statistics, client, href, depth - 1, fetched, &block)
+				end.each(&:wait)
+			rescue Async::TimeoutError
+				Async.logger.error(self) {"Timeout while fetching #{url}"}
+			rescue StandardError
+				Async.logger.error(self) {$!}
+			end
+			
+			sync def call(urls, &block)
+				statistics = Statistics.new
+				
+				urls.each do |url|
+					endpoint = Async::HTTP::Endpoint.parse(url, timeout: 10)
+					
+					Async::HTTP::Client.open(endpoint, endpoint.protocol, connection_limit: 4) do |client|
+						fetch(statistics, client, endpoint.url, &block).wait
+					end
+				end
+				
+				return statistics
+			end
+		end
+	end
+end

data/lib/benchmark/http/version.rb

@@ -20,6 +20,6 @@
 
 module Benchmark
 	module HTTP
-		VERSION = "0.9.0"
+		VERSION = "0.10.0"
 	end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: benchmark-http
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.10.0
 platform: ruby
 authors:
 - Samuel Williams
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-12-16 00:00:00.000000000 Z
+date: 2020-01-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: async-io
@@ -146,6 +146,7 @@ files:
 - lib/benchmark/http/command/spider.rb
 - lib/benchmark/http/links_filter.rb
 - lib/benchmark/http/seconds.rb
+- lib/benchmark/http/spider.rb
 - lib/benchmark/http/statistics.rb
 - lib/benchmark/http/version.rb
 homepage: https://github.com/socketry/benchmark-http
@@ -166,7 +167,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.4
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: An asynchronous benchmark toolbox for modern HTTP servers.