async-http 0.9.0 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 52cf6dc85fcbfa045a17959e27ba77d8cc3ebeb211074598501ebfeb560f3ac6
-  data.tar.gz: 34a8c1a0bfa896da593b901cc31eb80002d74e3d4422440f6ad24073156b009c
+  metadata.gz: c74bbff1b6216d57e8d60e5a54700f9461cef5aff2436db4abbf07c300a1af09
+  data.tar.gz: 1354a61e69f69706688f112fef4c958af285bec43172f00d7993bff7fcb9dc7b
 SHA512:
-  metadata.gz: 630b0308b56222426254806595c485f1fbe75b98f2e5a7b85c2d889176e28e04068054672e1b74a302680b15a972d1b3d46a002d03da10a66af5a937e5f63dcd
-  data.tar.gz: 17fe537532ce17b0e3103f1f645dd8c900a42aad8d1bffcca0a8da5fc4476debb924eb5319fc19a50e7ee6b2db5f0517eda3cab6b79ef94bedcedc9d529a1509
+  metadata.gz: d62959c37bc323c82d57ca4e745ef4d2a715f3ec0e580e0d4a1e5cbdb65766f6b4c671efab4f8e85fb0fabe6a5cac99be1de1aa2cc92c30bc39e5e51f9f40d51
+  data.tar.gz: 75b83edd5bd767e638ebeacbd3446e317b24b49eba17628af95e32e78690cc0b26c8c998ed1aaed7248c8e796c7e77fd43a8c80f391fcc5876a8cd84533bdd1d

data/Gemfile

@@ -4,9 +4,7 @@ source 'https://rubygems.org'
 gemspec
 
 group :development do
-	gem 'trenni-sanitize'
-	
-	gem 'async-await'
+	gem 'pry'
 end
 
 group :test do

data/async-http.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
 	spec.require_paths = ["lib"]
 	
 	spec.add_dependency("async", "~> 1.4")
-	spec.add_dependency("async-io", "~> 1.5")
+	spec.add_dependency("async-io", "~> 1.6")
 	
 	spec.add_dependency("http-2", "~> 0.8")
 	# spec.add_dependency("openssl")

data/lib/async/http/client.rb

@@ -25,14 +25,19 @@ require_relative 'protocol'
 module Async
 	module HTTP
 		class Client
-			def initialize(endpoint, protocol = Protocol::HTTPS, **options)
+			def initialize(endpoint, protocol = nil, authority = nil, **options)
 				@endpoint = endpoint
 				
-				@protocol = protocol
+				@protocol = protocol || endpoint.protocol
+				@authority = authority || endpoint.hostname
 				
 				@connections = connect(**options)
 			end
 			
+			attr :endpoint
+			attr :protocol
+			attr :authority
+			
 			def self.open(*args, &block)
 				client = self.new(*args)
 				
@@ -59,7 +64,7 @@ module Async
 			
 			def request(*args)
 				@connections.acquire do |connection|
-					connection.send_request(*args)
+					connection.send_request(@authority, *args)
 				end
 			end
 			

data/lib/async/http/headers.rb

@@ -0,0 +1,85 @@
+# Copyright, 2017, by Samuel G. D. Williams. <http://www.codeotaku.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+module Async
+	module HTTP
+		class Headers
+			def initialize
+				@hash = {}
+			end
+			
+			def freeze
+				return unless frozen?
+				
+				@hash.freeze
+				
+				super
+			end
+			
+			def inspect
+				@hash.inspect
+			end
+			
+			def []= key, value
+				@hash[symbolize(key)] = value
+			end
+			
+			def [] key
+				@hash[key]
+			end
+			
+			def == other
+				@hash == other.to_hash
+			end
+			
+			def delete(key)
+				@hash.delete(key)
+			end
+			
+			def each
+				return to_enum unless block_given?
+				
+				@hash.each do |key, value|
+					yield stringify(key), value
+				end
+			end
+			
+			def symbolize(value)
+				Headers[value]
+			end
+			
+			def stringify(key)
+				key.to_s.tr('_', '-')
+			end
+			
+			def to_hash
+				@hash
+			end
+			
+			def to_http_hash
+				Hash[@hash.map{|key, value| ["HTTP_#{key.to_s.upcase}", value]}]
+			end
+			
+			def self.[] value
+				value.downcase.tr('-', '_').to_sym
+			end
+		end
+	end
+end

data/lib/async/http/pool.rb

@@ -112,8 +112,10 @@ module Async
 					end
 				end
 				
-				Async.logger.debug(self) {"No available resources, allocating new one..."}
-				return create_resource
+				if !@limit or @available.count < @limit
+					Async.logger.debug(self) {"No available resources, allocating new one..."}
+					return create_resource
+				end
 			end
 		end
 	end

data/lib/async/http/protocol/http10.rb

@@ -35,7 +35,7 @@ module Async
 				end
 				
 				def keep_alive?(headers)
-					headers[HTTP_CONNECTION] == KEEP_ALIVE
+					headers[:connection] == KEEP_ALIVE
 				end
 				
 				# Server loop.
@@ -64,7 +64,7 @@ module Async
 				end
 				
 				def read_body(headers)
-					if content_length = headers[HTTP_CONTENT_LENGTH]
+					if content_length = headers[:content_length]
 						return @stream.read(Integer(content_length))
 					# elsif !keep_alive?(headers)
 					# 	return @stream.read

data/lib/async/http/protocol/http11.rb

@@ -22,14 +22,15 @@ require 'async/io/protocol/line'
 
 require_relative 'request'
 require_relative 'response'
+require_relative '../headers'
 
 module Async
 	module HTTP
 		module Protocol
 			# Implements basic HTTP/1.1 request/response.
 			class HTTP11 < Async::IO::Protocol::Line
-				HTTP_CONTENT_LENGTH = 'HTTP_CONTENT_LENGTH'.freeze
-				HTTP_TRANSFER_ENCODING = 'HTTP_TRANSFER_ENCODING'.freeze
+				CONTENT_LENGTH = Headers['Content-Length']
+				TRANSFER_ENCODING = Headers['Transfer-Encoding']
 				
 				CRLF = "\r\n".freeze
 				
@@ -53,7 +54,6 @@ module Async
 					alias client new
 				end
 				
-				HTTP_CONNECTION = 'HTTP_CONNECTION'.freeze
 				KEEP_ALIVE = 'keep-alive'.freeze
 				CLOSE = 'close'.freeze
 				
@@ -64,7 +64,7 @@ module Async
 				end
 				
 				def keep_alive?(headers)
-					headers[HTTP_CONNECTION] != CLOSE
+					headers[:connection] != CLOSE
 				end
 				
 				# Server loop.
@@ -88,16 +88,20 @@ module Async
 				end
 				
 				# Client request.
-				def send_request(method, path, headers = {}, body = [])
-					write_request(method, path, version, headers, body)
+				def send_request(authority, method, path, headers = {}, body = [])
+					Async.logger.debug(self) {"#{method} #{path} #{headers.inspect}"}
+					
+					write_request(authority, method, path, version, headers, body)
 					
 					return Response.new(*read_response)
 				rescue EOFError
 					return nil
 				end
 				
-				def write_request(method, path, version, headers, body)
+				def write_request(authority, method, path, version, headers, body)
 					@stream.write("#{method} #{path} #{version}\r\n")
+					@stream.write("Host: #{authority}\r\n")
+					
 					write_headers(headers)
 					write_body(body)
 					
@@ -121,7 +125,7 @@ module Async
 					headers = read_headers
 					body = read_body(headers)
 					
-					return method, path, version, headers, body
+					return headers.delete(:host), method, path, version, headers, body
 				end
 				
 				def write_response(version, status, headers, body)
@@ -142,11 +146,11 @@ module Async
 					end
 				end
 				
-				def read_headers(headers = {})
+				def read_headers(headers = Headers.new)
 					# Parsing headers:
 					each_line do |line|
 						if line =~ /^([a-zA-Z\-]+):\s*(.+?)\s*$/
-							headers["HTTP_#{$1.tr('-', '_').upcase}"] = $2
+							headers[$1] = $2
 						else
 							break
 						end
@@ -178,7 +182,7 @@ module Async
 				end
 				
 				def read_body(headers)
-					if headers[HTTP_TRANSFER_ENCODING] == 'chunked'
+					if headers[:transfer_encoding] == 'chunked'
 						buffer = Async::IO::BinaryString.new
 						
 						while true
@@ -195,7 +199,7 @@ module Async
 						end
 						
 						return buffer
-					elsif content_length = headers[HTTP_CONTENT_LENGTH]
+					elsif content_length = headers[:content_length]
 						return @stream.read(Integer(content_length))
 					end
 				end

data/lib/async/http/protocol/http2.rb

@@ -20,6 +20,7 @@
 
 require_relative 'request'
 require_relative 'response'
+require_relative '../headers'
 
 require 'async/notification'
 
@@ -38,6 +39,14 @@ module Async
 					self.new(::HTTP2::Server.new, stream)
 				end
 				
+				HTTPS = 'https'.freeze
+				SCHEME = ':scheme'.freeze
+				METHOD = ':method'.freeze
+				PATH = ':path'.freeze
+				AUTHORITY = ':authority'.freeze
+				REASON = ':reason'.freeze
+				STATUS = ':status'.freeze
+				
 				def initialize(controller, stream)
 					@controller = controller
 					@stream = stream
@@ -72,15 +81,13 @@ module Async
 				
 				def read_in_background(task: Task.current)
 					task.async do |nested_task|
-						while true
-							if data = @stream.io.read(10)
-								# Async.logger.debug(self) {"Reading data: #{data.size} bytes"}
-								@controller << data
-							else
-								Async.logger.debug(self) {"Connection reset by peer!"}
-								break
-							end
+						buffer = Async::IO::BinaryString.new
+						
+						while data = @stream.io.read(1024*8, buffer)
+							@controller << data
 						end
+						
+						Async.logger.debug(self) {"Connection reset by peer!"}
 					end
 				end
 				
@@ -95,17 +102,19 @@ module Async
 					@controller.on(:stream) do |stream|
 						request = Request.new
 						request.version = "HTTP/2.0"
-						request.headers = {}
+						request.headers = Headers.new
 						
 						# stream.on(:active) { } # fires when stream transitions to open state
 						# stream.on(:close) { } # stream is closed by client and server
 						
 						stream.on(:headers) do |headers|
 							headers.each do |key, value|
-								if key == ':method'
+								if key == METHOD
 									request.method = value
-								elsif key == ':path'
+								elsif key == PATH
 									request.path = value
+								elsif key == AUTHORITY
+									request.authority = value
 								else
 									request.headers[key] = value
 								end
@@ -120,7 +129,7 @@ module Async
 							response = yield request
 							
 							# send response
-							stream.headers(':status' => response[0].to_s)
+							stream.headers(STATUS => response[0].to_s)
 							
 							stream.headers(response[1]) unless response[1].empty?
 							
@@ -137,13 +146,16 @@ module Async
 					end
 				end
 				
-				def send_request(method, path, headers = {}, body = nil)
+				RESPONSE_VERSION = 'HTTP/2'.freeze
+				
+				def send_request(authority, method, path, headers = {}, body = nil)
 					stream = @controller.new_stream
 					
 					internal_headers = {
-						':scheme' => 'https',
-						':method' => method,
-						':path' => path,
+						SCHEME => HTTPS,
+						METHOD => method,
+						PATH => path,
+						AUTHORITY => authority,
 					}.merge(headers)
 					
 					stream.headers(internal_headers, end_stream: true)
@@ -157,17 +169,17 @@ module Async
 					# end
 					
 					response = Response.new
-					response.version = "HTTP/2"
-					response.headers = {}
+					response.version = RESPONSE_VERSION
+					response.headers = Headers.new
 					response.body = Async::IO::BinaryString.new
 					
 					stream.on(:headers) do |headers|
 						# Async.logger.debug(self) {"Stream headers: #{headers.inspect}"}
 						
 						headers.each do |key, value|
-							if key == ':status'
+							if key == STATUS
 								response.status = value.to_i
-							elsif key == ':reason'
+							elsif key == REASON
 								response.reason = value
 							else
 								response.headers[key] = value

data/lib/async/http/protocol/request.rb

@@ -21,7 +21,7 @@
 module Async
 	module HTTP
 		module Protocol
-			class Request < Struct.new(:method, :path, :version, :headers, :body)
+			class Request < Struct.new(:authority, :method, :path, :version, :headers, :body)
 				def read
 					@body
 				end

data/lib/async/http/server.rb

@@ -34,7 +34,7 @@ module Async
 				[200, {"Content-Type" => "text/plain"}, ["Hello World"]]
 			end
 			
-			def accept(peer, address)
+			def accept(peer, address, task: Task.current)
 				stream = Async::IO::Stream.new(peer)
 				protocol = @protocol_class.server(stream)
 				

data/lib/async/http/url_endpoint.rb

@@ -31,9 +31,14 @@ module Async
 			end
 			
 			def initialize(url, endpoint = nil, **options)
+				super(**options)
+				
 				@url = url
 				@endpoint = endpoint
-				@options = options
+			end
+			
+			def to_s
+				"\#<#{self.class} #{@url} #{@options.inspect}>"
 			end
 			
 			attr :url
@@ -64,7 +69,7 @@ module Async
 			end
 			
 			def hostname
-				@url.hostname
+				@options.fetch(:hostname, @url.hostname)
 			end
 			
 			def ssl_context
@@ -79,8 +84,6 @@ module Async
 					@endpoint = Async::IO::Endpoint.tcp(hostname, port)
 					
 					if secure?
-						Async.logger.debug(self) {"Setting hostname: #{self.hostname}"}
-						
 						# Wrap it in SSL:
 						@endpoint = Async::IO::SecureEndpoint.new(@endpoint,
 							ssl_context: ssl_context,

data/lib/async/http/version.rb

@@ -20,6 +20,6 @@
 
 module Async
 	module HTTP
-		VERSION = "0.9.0"
+		VERSION = "0.10.0"
 	end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: async-http
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.10.0
 platform: ruby
 authors:
 - Samuel Williams
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-27 00:00:00.000000000 Z
+date: 2018-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: async
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '1.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '1.6'
 - !ruby/object:Gem::Dependency
   name: http-2
   requirement: !ruby/object:Gem::Requirement
@@ -123,9 +123,9 @@ files:
 - README.md
 - Rakefile
 - async-http.gemspec
-- examples/spider.rb
 - lib/async/http.rb
 - lib/async/http/client.rb
+- lib/async/http/headers.rb
 - lib/async/http/pool.rb
 - lib/async/http/protocol.rb
 - lib/async/http/protocol/http1.rb

data/examples/spider.rb

@@ -1,129 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'async/await'
-
-require 'pry'
-
-require_relative '../lib/async/http/client'
-require '../lib/async/http/url_endpoint'
-require '../lib/async/http/protocol/https'
-
-require 'trenni/sanitize'
-require 'set'
-
-Async.logger.level = Logger::DEBUG
-
-class HTML < Trenni::Sanitize::Filter
-	def initialize(*)
-		super
-		
-		@base = nil
-		@links = []
-	end
-	
-	attr :base
-	attr :links
-	
-	def filter(node)
-		if node.name == 'base'
-			@base = node['href']
-		elsif node.name == 'a'
-			@links << node['href']
-		end
-		
-		node.skip!(TAG)
-	end
-end
-
-class Cache
-	def initialize
-		@clients = {}
-	end
-	
-	def close
-		@clients.each(&:close)
-		@clients.clear
-	end
-	
-	def [] endpoint
-		url = endpoint.specification
-		key = "#{url.scheme}://#{url.userinfo}@#{url.hostname}"
-		
-		@clients[key] ||= Async::HTTP::Client.new(endpoint, endpoint.secure? ? Async::HTTP::Protocol::HTTPS : Async::HTTP::Protocol::HTTP1)
-	end
-end
-
-class << self
-	include Async::Await
-	
-	async def fetch(url, depth = 4, fetched = Set.new, clients = Cache.new)
-		return if fetched.include?(url) or depth == 0 or url.host != "www.codeotaku.com"
-		fetched << url
-		
-		endpoint = Async::HTTP::URLEndpoint.new(url)
-		client = clients[endpoint]
-		
-		request_uri = endpoint.specification.request_uri
-		puts "GET #{url} (depth = #{depth})"
-		
-		response = timeout(10) do
-			client.get(request_uri, {
-				':authority' => endpoint.specification.hostname,
-				'accept' => '*/*',
-				'user-agent' => 'spider',
-			})
-		end
-		
-		if response.status >= 300 && response.status < 400
-			location = url + response.headers['location']
-			# puts "Following redirect to #{location}"
-			return fetch(location, depth-1, fetched)
-		end
-		
-		content_type = response.headers['content-type']
-		unless content_type&.start_with? 'text/html'
-			# puts "Unsupported content type: #{response.headers['content-type']}"
-			return
-		end
-		
-		base = endpoint.specification
-		
-		begin
-			html = HTML.parse(response.body)
-		rescue
-			# Async.logger.error($!)
-			return
-		end
-		
-		if html.base
-			base = base + html.base
-		end
-		
-		html.links.each do |href|
-			begin
-				full_url = base + href
-				
-				fetch(full_url, depth - 1, fetched) if full_url.kind_of? URI::HTTP
-			rescue ArgumentError, URI::InvalidURIError
-				# puts "Could not fetch #{href}, relative to #{base}."
-			end
-		end
-	rescue Async::TimeoutError
-		Async.logger.error("Timeout while fetching #{url}")
-	rescue StandardError
-		Async.logger.error($!)
-	ensure
-		puts "Closing client from spider..."
-		client.close if client
-	end
-	
-	async def fetch_one(url)
-		endpoint = Async::HTTP::URLEndpoint.new(url)
-		client = Async::HTTP::Client.new(endpoint, endpoint.secure? ? Async::HTTP::Protocol::HTTPS : Async::HTTP::Protocol::HTTP1)
-		
-		binding.pry
-	end
-end
-
-fetch_one(URI.parse("https://www.codeotaku.com"))
-#puts "Finished."