async-http 0.52.3 → 0.52.4

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 2b5f422d50d91e0a55cf4a231bab122661198bf277ef483c5d342feb96f5e5d0
4
- data.tar.gz: acf9ea6b2d1c332b73ead3f4203725cc3a750418ceb945a8ca1f431c71e8cfba
3
+ metadata.gz: 99b5cbef0cb74dedeea089ac42ff1aa5315e1089079f812ff4120e9147ab22ec
4
+ data.tar.gz: dda68448abfa958e9a98814f35482bda367990191f9742292512c8f242745fe8
5
5
  SHA512:
6
- metadata.gz: 9744602130e3f4587f9e56e26cff4db9656b36685ae4ca6d15275e858c27bc239bec304e7146620231a0bb1619f11dea6153ccb41573caa80f94a4904f1a5edc
7
- data.tar.gz: 4907eba09d6899a0bb97b59c3f75e35bed686b3b103107d69b96298f0358a7bb188f1358f4a799fe97c8837a34044af74ea5ba7728b9eb86592f208d1091b505
6
+ metadata.gz: 74ddbd13c0e1b3c5bae03ded50129ab73fac1285d480bef0ff8a944c1163651037d6a258f36757b8969f72ea7e63eaf145384b346aa7733f2a51430c85cba642
7
+ data.tar.gz: e1036b972800356861167204b1ecfbab6e164dbe51351bd15a2881214d5e437f1cda9bd53a75715bd41c74f91ea6911d438b2ebcc04adceede636d0814e79e1f
@@ -0,0 +1,71 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'csv'
4
+ require 'json'
5
+ require 'net/http'
6
+
7
+ require 'protocol/http/header/authorization'
8
+
9
+ class RateLimitingError < StandardError; end
10
+
11
+ @user = ENV['GITHUB_USER']
12
+ @token = ENV['GITHUB_TOKEN']
13
+
14
+ unless @user && @token
15
+ fail "export GITHUB_USER and GITHUB_TOKEN!"
16
+ end
17
+
18
+ def fetch_github_license(homepage_uri)
19
+ %r{github.com/(?<owner>.+?)/(?<repo>.+)} =~ homepage_uri
20
+ return nil unless repo
21
+
22
+ url = URI.parse("https://api.github.com/repos/#{owner}/#{repo}/license")
23
+ request = Net::HTTP::Get.new(url)
24
+
25
+ request['user-agent'] = 'fetch-github-licenses'
26
+ request['authorization'] = Protocol::HTTP::Header::Authorization.basic(@user, @token)
27
+
28
+ response = Net::HTTP.start(url.hostname) do |http|
29
+ http.request(request)
30
+ end
31
+
32
+ case response
33
+ when Net::HTTPOK
34
+ JSON.parse(response.body).dig('license', 'spdx_id')
35
+ when Net::HTTPNotFound, Net::HTTPMovedPermanently, Net::HTTPForbidden
36
+ nil
37
+ else
38
+ raise response.body
39
+ end
40
+ end
41
+
42
+ def fetch_rubygem_license(name, version)
43
+ url = URI.parse("https://rubygems.org/api/v2/rubygems/#{name}/versions/#{version}.json")
44
+ response = Net::HTTP.get_response(url)
45
+
46
+ case response
47
+ when Net::HTTPOK
48
+ body = JSON.parse(response.body)
49
+ [name, body.dig('licenses', 0) || fetch_github_license(body['homepage_uri'])]
50
+ when Net::HTTPNotFound
51
+ [name, nil] # from a non rubygems remote
52
+ when Net::HTTPTooManyRequests
53
+ raise RateLimitingError
54
+ else
55
+ raise response.body
56
+ end
57
+ rescue RateLimitingError
58
+ sleep 1
59
+
60
+ retry
61
+ end
62
+
63
+ threads = ARGF.map do |line|
64
+ if line == "GEM\n" .. line.chomp.empty?
65
+ /\A\s{4}(?<name>[a-z].+?) \((?<version>.+)\)\n\z/ =~ line
66
+
67
+ Thread.new { fetch_rubygem_license(name, version) } if name
68
+ end
69
+ end.compact
70
+
71
+ puts CSV.generate { |csv| threads.each { csv << _1.value } }
@@ -0,0 +1,90 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'csv'
4
+ require 'json'
5
+ require 'async/http/internet'
6
+
7
+ class RateLimitingError < StandardError; end
8
+
9
+ @internet = Async::HTTP::Internet.new
10
+
11
+ @user = ENV['GITHUB_USER']
12
+ @token = ENV['GITHUB_TOKEN']
13
+
14
+ unless @user && @token
15
+ fail "export GITHUB_USER and GITHUB_TOKEN!"
16
+ end
17
+
18
+ GITHUB_HEADERS = {
19
+ 'user-agent' => 'fetch-github-licenses',
20
+ 'authorization' => Protocol::HTTP::Header::Authorization.basic(@user, @token)
21
+ }
22
+
23
+ RUBYGEMS_HEADERS = {
24
+ 'user-agent' => 'fetch-github-licenses'
25
+ }
26
+
27
+ def fetch_github_license(homepage_uri)
28
+ %r{github.com/(?<owner>.+?)/(?<repo>.+)} =~ homepage_uri
29
+ return nil unless repo
30
+
31
+ response = @internet.get("https://api.github.com/repos/#{owner}/#{repo}/license", GITHUB_HEADERS)
32
+
33
+ case response.status
34
+ when 200
35
+ return JSON.parse(response.read).dig('license', 'spdx_id')
36
+ when 404
37
+ return nil
38
+ else
39
+ raise response.read
40
+ end
41
+ ensure
42
+ response.finish
43
+ end
44
+
45
+ def fetch_rubygem_license(name, version)
46
+ response = @internet.get("https://rubygems.org/api/v2/rubygems/#{name}/versions/#{version}.json", RUBYGEMS_HEADERS)
47
+
48
+ case response.status
49
+ when 200
50
+ body = JSON.parse(response.read)
51
+ [name, body.dig('licenses', 0) || fetch_github_license(body['homepage_uri'])]
52
+ when 404
53
+ [name, nil] # from a non rubygems remote
54
+ when 429
55
+ raise RateLimitingError
56
+ else
57
+ raise response.read
58
+ end
59
+ rescue RateLimitingError
60
+ response.finish
61
+
62
+ Async.logger.warn(name) {"Rate limited..."}
63
+ Async::Task.current.sleep(1.0)
64
+
65
+ retry
66
+ ensure
67
+ response.finish
68
+ end
69
+
70
+ Sync do |parent|
71
+ output = CSV.new($stdout)
72
+
73
+ tasks = ARGF.map do |line|
74
+ if line == "GEM\n" .. line.chomp.empty?
75
+ /\A\s{4}(?<name>[a-z].+?) \((?<version>.+)\)\n\z/ =~ line
76
+
77
+ parent.async do
78
+ fetch_rubygem_license(name, version)
79
+ end if name
80
+ end
81
+ end.compact
82
+
83
+ tasks.each do |task|
84
+ output << task.wait
85
+ end
86
+
87
+ @internet.instance_variable_get(:@clients).each do |name, client|
88
+ puts client.pool
89
+ end
90
+ end
@@ -37,8 +37,8 @@ module Async
37
37
 
38
38
  # In order for this implementation to work correctly, you need to use a LimitedQueue.
39
39
  # @param minimum_throughput [Integer] the minimum bytes per second otherwise this body will be forcefully closed.
40
- def initialize(*args, minimum_throughput: 1024, **options)
41
- super(*args, **options)
40
+ def initialize(*arguments, minimum_throughput: 1024, **options)
41
+ super(*arguments, **options)
42
42
 
43
43
  @minimum_throughput = minimum_throughput
44
44
 
@@ -193,8 +193,8 @@ module Async
193
193
  @endpoint ||= build_endpoint
194
194
  end
195
195
 
196
- def bind(*args, &block)
197
- endpoint.bind(*args, &block)
196
+ def bind(*arguments, &block)
197
+ endpoint.bind(*arguments, &block)
198
198
  end
199
199
 
200
200
  def connect(&block)
@@ -33,6 +33,10 @@ module Async
33
33
  @options = options
34
34
  end
35
35
 
36
+ # A cache of clients.
37
+ # @attribute [Hash(URI, Client)]
38
+ attr :clients
39
+
36
40
  def call(method, url, headers = nil, body = nil)
37
41
  endpoint = Endpoint.parse(url)
38
42
  key = host_key(endpoint)
@@ -56,7 +56,7 @@ module Async
56
56
 
57
57
  # Server loop.
58
58
  def each(task: Task.current)
59
- task.annotate("Reading #{version} requests for #{self.class}.")
59
+ task.annotate("Reading #{self.version} requests for #{self.class}.")
60
60
 
61
61
  while request = next_request
62
62
  response = yield(request, self)
@@ -90,11 +90,12 @@ module Async
90
90
  body.call(stream)
91
91
  else
92
92
  head = request.head?
93
+ version = request.version
93
94
 
94
95
  request = nil unless body
95
96
  response = nil
96
97
 
97
- write_body(request.version, body, head, trailers)
98
+ write_body(version, body, head, trailers)
98
99
  end
99
100
  else
100
101
  # If the request failed to generate a response, it was an internal server error:
@@ -22,6 +22,6 @@
22
22
 
23
23
  module Async
24
24
  module HTTP
25
- VERSION = "0.52.3"
25
+ VERSION = "0.52.4"
26
26
  end
27
27
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: async-http
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.52.3
4
+ version: 0.52.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Samuel Williams
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-05-08 00:00:00.000000000 Z
11
+ date: 2020-05-21 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: async
@@ -219,6 +219,8 @@ files:
219
219
  - examples/fetch/public/index.html
220
220
  - examples/fetch/public/stream.js
221
221
  - examples/google/search.rb
222
+ - examples/licenses/gemspect.rb
223
+ - examples/licenses/list.rb
222
224
  - examples/request.rb
223
225
  - examples/stream/stop.rb
224
226
  - examples/trenni/Gemfile