async-http 0.52.3 → 0.52.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2b5f422d50d91e0a55cf4a231bab122661198bf277ef483c5d342feb96f5e5d0
-  data.tar.gz: acf9ea6b2d1c332b73ead3f4203725cc3a750418ceb945a8ca1f431c71e8cfba
+  metadata.gz: 99b5cbef0cb74dedeea089ac42ff1aa5315e1089079f812ff4120e9147ab22ec
+  data.tar.gz: dda68448abfa958e9a98814f35482bda367990191f9742292512c8f242745fe8
 SHA512:
-  metadata.gz: 9744602130e3f4587f9e56e26cff4db9656b36685ae4ca6d15275e858c27bc239bec304e7146620231a0bb1619f11dea6153ccb41573caa80f94a4904f1a5edc
-  data.tar.gz: 4907eba09d6899a0bb97b59c3f75e35bed686b3b103107d69b96298f0358a7bb188f1358f4a799fe97c8837a34044af74ea5ba7728b9eb86592f208d1091b505
+  metadata.gz: 74ddbd13c0e1b3c5bae03ded50129ab73fac1285d480bef0ff8a944c1163651037d6a258f36757b8969f72ea7e63eaf145384b346aa7733f2a51430c85cba642
+  data.tar.gz: e1036b972800356861167204b1ecfbab6e164dbe51351bd15a2881214d5e437f1cda9bd53a75715bd41c74f91ea6911d438b2ebcc04adceede636d0814e79e1f

data/examples/licenses/gemspect.rb

@@ -0,0 +1,71 @@
+#!/usr/bin/env ruby
+
+require 'csv'
+require 'json'
+require 'net/http'
+
+require 'protocol/http/header/authorization'
+
+class RateLimitingError < StandardError; end
+
+@user = ENV['GITHUB_USER']
+@token = ENV['GITHUB_TOKEN']
+
+unless @user && @token
+	fail "export GITHUB_USER and GITHUB_TOKEN!"
+end
+
+def fetch_github_license(homepage_uri)
+	%r{github.com/(?<owner>.+?)/(?<repo>.+)} =~ homepage_uri
+	return nil unless repo
+
+	url = URI.parse("https://api.github.com/repos/#{owner}/#{repo}/license")
+	request = Net::HTTP::Get.new(url)
+
+	request['user-agent'] = 'fetch-github-licenses'
+	request['authorization'] = Protocol::HTTP::Header::Authorization.basic(@user, @token)
+
+	response = Net::HTTP.start(url.hostname) do |http|
+		http.request(request)
+	end
+
+	case response
+	when Net::HTTPOK
+		JSON.parse(response.body).dig('license', 'spdx_id')
+	when Net::HTTPNotFound, Net::HTTPMovedPermanently, Net::HTTPForbidden
+		nil
+	else
+		raise response.body
+	end
+end
+
+def fetch_rubygem_license(name, version)
+	url = URI.parse("https://rubygems.org/api/v2/rubygems/#{name}/versions/#{version}.json")
+	response = Net::HTTP.get_response(url)
+
+	case response
+	when Net::HTTPOK
+		body = JSON.parse(response.body)
+		[name, body.dig('licenses', 0) || fetch_github_license(body['homepage_uri'])]
+	when Net::HTTPNotFound
+		[name, nil] # from a non rubygems remote
+	when Net::HTTPTooManyRequests
+		raise RateLimitingError
+	else
+		raise response.body
+	end
+rescue RateLimitingError
+	sleep 1
+	
+	retry
+end
+
+threads = ARGF.map do |line|
+	if line == "GEM\n" .. line.chomp.empty?
+		/\A\s{4}(?<name>[a-z].+?) \((?<version>.+)\)\n\z/ =~ line
+		
+		Thread.new { fetch_rubygem_license(name, version) } if name
+	end
+end.compact
+
+puts CSV.generate { |csv| threads.each { csv << _1.value } }

data/examples/licenses/list.rb

@@ -0,0 +1,90 @@
+#!/usr/bin/env ruby
+
+require 'csv'
+require 'json'
+require 'async/http/internet'
+
+class RateLimitingError < StandardError; end
+
+@internet = Async::HTTP::Internet.new
+
+@user = ENV['GITHUB_USER']
+@token = ENV['GITHUB_TOKEN']
+
+unless @user && @token
+	fail "export GITHUB_USER and GITHUB_TOKEN!"
+end
+
+GITHUB_HEADERS = {
+	'user-agent' => 'fetch-github-licenses',
+	'authorization' => Protocol::HTTP::Header::Authorization.basic(@user, @token)
+}
+
+RUBYGEMS_HEADERS = {
+	'user-agent' => 'fetch-github-licenses'
+}
+
+def fetch_github_license(homepage_uri)
+	%r{github.com/(?<owner>.+?)/(?<repo>.+)} =~ homepage_uri
+	return nil unless repo
+
+	response = @internet.get("https://api.github.com/repos/#{owner}/#{repo}/license", GITHUB_HEADERS)
+	
+	case response.status
+	when 200
+		return JSON.parse(response.read).dig('license', 'spdx_id')
+	when 404
+		return nil
+	else
+		raise response.read
+	end
+ensure
+	response.finish
+end
+
+def fetch_rubygem_license(name, version)
+	response = @internet.get("https://rubygems.org/api/v2/rubygems/#{name}/versions/#{version}.json", RUBYGEMS_HEADERS)
+	
+	case response.status
+	when 200
+		body = JSON.parse(response.read)
+		[name, body.dig('licenses', 0) || fetch_github_license(body['homepage_uri'])]
+	when 404
+		[name, nil] # from a non rubygems remote
+	when 429
+		raise RateLimitingError
+	else
+		raise response.read
+	end
+rescue RateLimitingError
+	response.finish
+	
+	Async.logger.warn(name) {"Rate limited..."}
+	Async::Task.current.sleep(1.0)
+	
+	retry
+ensure
+	response.finish
+end
+
+Sync do |parent|
+	output = CSV.new($stdout)
+	
+	tasks = ARGF.map do |line|
+		if line == "GEM\n" .. line.chomp.empty?
+			/\A\s{4}(?<name>[a-z].+?) \((?<version>.+)\)\n\z/ =~ line
+			
+			parent.async do
+				fetch_rubygem_license(name, version)
+			end if name
+		end
+	end.compact
+	
+	tasks.each do |task|
+		output << task.wait
+	end
+	
+	@internet.instance_variable_get(:@clients).each do |name, client|
+		puts client.pool
+	end
+end

data/lib/async/http/body/slowloris.rb

@@ -37,8 +37,8 @@ module Async
 				
 				# In order for this implementation to work correctly, you need to use a LimitedQueue.
 				# @param minimum_throughput [Integer] the minimum bytes per second otherwise this body will be forcefully closed.
-				def initialize(*args, minimum_throughput: 1024, **options)
-					super(*args, **options)
+				def initialize(*arguments, minimum_throughput: 1024, **options)
+					super(*arguments, **options)
 					
 					@minimum_throughput = minimum_throughput
 					

data/lib/async/http/endpoint.rb

@@ -193,8 +193,8 @@ module Async
 				@endpoint ||= build_endpoint
 			end
 			
-			def bind(*args, &block)
-				endpoint.bind(*args, &block)
+			def bind(*arguments, &block)
+				endpoint.bind(*arguments, &block)
 			end
 			
 			def connect(&block)

data/lib/async/http/internet.rb

@@ -33,6 +33,10 @@ module Async
 				@options = options
 			end
 			
+			# A cache of clients.
+			# @attribute [Hash(URI, Client)]
+			attr :clients
+			
 			def call(method, url, headers = nil, body = nil)
 				endpoint = Endpoint.parse(url)
 				key = host_key(endpoint)

data/lib/async/http/protocol/http1/server.rb

@@ -56,7 +56,7 @@ module Async
 					
 					# Server loop.
 					def each(task: Task.current)
-						task.annotate("Reading #{version} requests for #{self.class}.")
+						task.annotate("Reading #{self.version} requests for #{self.class}.")
 						
 						while request = next_request
 							response = yield(request, self)
@@ -90,11 +90,12 @@ module Async
 									body.call(stream)
 								else
 									head = request.head?
+									version = request.version
 									
 									request = nil unless body
 									response = nil
 									
-									write_body(request.version, body, head, trailers)
+									write_body(version, body, head, trailers)
 								end
 							else
 								# If the request failed to generate a response, it was an internal server error:

data/lib/async/http/version.rb

@@ -22,6 +22,6 @@
 
 module Async
 	module HTTP
-		VERSION = "0.52.3"
+		VERSION = "0.52.4"
 	end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: async-http
 version: !ruby/object:Gem::Version
-  version: 0.52.3
+  version: 0.52.4
 platform: ruby
 authors:
 - Samuel Williams
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-08 00:00:00.000000000 Z
+date: 2020-05-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: async
@@ -219,6 +219,8 @@ files:
 - examples/fetch/public/index.html
 - examples/fetch/public/stream.js
 - examples/google/search.rb
+- examples/licenses/gemspect.rb
+- examples/licenses/list.rb
 - examples/request.rb
 - examples/stream/stop.rb
 - examples/trenni/Gemfile