gemstash 2.8.2 → 2.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. checksums.yaml +4 -4
  2. data/lib/gemstash/authorization.rb +0 -2
  3. data/lib/gemstash/cache.rb +5 -4
  4. data/lib/gemstash/cli/base.rb +9 -2
  5. data/lib/gemstash/cli/setup.rb +1 -1
  6. data/lib/gemstash/cli/start.rb +4 -0
  7. data/lib/gemstash/configuration.rb +71 -1
  8. data/lib/gemstash/db/upstream.rb +30 -2
  9. data/lib/gemstash/dependencies.rb +28 -5
  10. data/lib/gemstash/env.rb +4 -2
  11. data/lib/gemstash/gem_fetcher.rb +1 -2
  12. data/lib/gemstash/gem_pusher.rb +24 -20
  13. data/lib/gemstash/gem_source/private_source.rb +22 -6
  14. data/lib/gemstash/gem_source/upstream_source.rb +129 -34
  15. data/lib/gemstash/http_client.rb +167 -8
  16. data/lib/gemstash/logging.rb +3 -1
  17. data/lib/gemstash/man/gemstash-authorize.1 +29 -48
  18. data/lib/gemstash/man/gemstash-authorize.1.txt +14 -14
  19. data/lib/gemstash/man/gemstash-configuration.5 +231 -154
  20. data/lib/gemstash/man/gemstash-configuration.5.txt +208 -93
  21. data/lib/gemstash/man/gemstash-customize.7 +150 -168
  22. data/lib/gemstash/man/gemstash-customize.7.txt +80 -86
  23. data/lib/gemstash/man/gemstash-debugging.7 +20 -33
  24. data/lib/gemstash/man/gemstash-debugging.7.txt +5 -7
  25. data/lib/gemstash/man/gemstash-deploy.7 +31 -53
  26. data/lib/gemstash/man/gemstash-deploy.7.txt +17 -18
  27. data/lib/gemstash/man/gemstash-mirror.7 +24 -37
  28. data/lib/gemstash/man/gemstash-mirror.7.txt +14 -9
  29. data/lib/gemstash/man/gemstash-multiple-sources.7 +105 -110
  30. data/lib/gemstash/man/gemstash-multiple-sources.7.txt +62 -41
  31. data/lib/gemstash/man/gemstash-private-gems.7 +87 -136
  32. data/lib/gemstash/man/gemstash-private-gems.7.txt +30 -30
  33. data/lib/gemstash/man/gemstash-readme.7 +105 -127
  34. data/lib/gemstash/man/gemstash-readme.7.txt +43 -49
  35. data/lib/gemstash/man/gemstash-setup.1 +22 -42
  36. data/lib/gemstash/man/gemstash-setup.1.txt +12 -12
  37. data/lib/gemstash/man/gemstash-start.1 +9 -27
  38. data/lib/gemstash/man/gemstash-start.1.txt +5 -5
  39. data/lib/gemstash/man/gemstash-status.1 +9 -27
  40. data/lib/gemstash/man/gemstash-status.1.txt +5 -5
  41. data/lib/gemstash/man/gemstash-stop.1 +9 -27
  42. data/lib/gemstash/man/gemstash-stop.1.txt +5 -5
  43. data/lib/gemstash/man/gemstash-version.1 +8 -28
  44. data/lib/gemstash/man/gemstash-version.1.txt +2 -2
  45. data/lib/gemstash/rack_env_rewriter.rb +3 -0
  46. data/lib/gemstash/upstream.rb +83 -6
  47. data/lib/gemstash/version.rb +1 -1
  48. data/lib/gemstash/web.rb +12 -0
  49. metadata +7 -7
@@ -7,6 +7,10 @@ module Gemstash
7
7
  module GemSource
8
8
  # GemSource that purely redirects to the upstream server.
9
9
  class RedirectSource < Gemstash::GemSource::Base
10
+ include Gemstash::Env::Helper
11
+
12
+ VALID_ID = /\A(?!\.+\z)[A-Za-z0-9._-]+\z/
13
+
10
14
  def self.rack_env_rewriter
11
15
  @rack_env_rewriter ||= Gemstash::RackEnvRewriter.new(%r{\A/redirect/(?<upstream_url>[^/]+)})
12
16
  end
@@ -16,7 +20,7 @@ module Gemstash
16
20
  return false unless rewriter.matches?
17
21
 
18
22
  rewriter.rewrite
19
- env["gemstash.upstream"] = rewriter.captures["upstream_url"]
23
+ env["gemstash.upstream"] = CGI.unescape(rewriter.captures["upstream_url"])
20
24
  capture_user_agent(env)
21
25
  true
22
26
  end
@@ -25,9 +29,14 @@ module Gemstash
25
29
  env["gemstash.user-agent"] = env["HTTP_USER_AGENT"]
26
30
  end
27
31
 
32
+ def redirect(target)
33
+ headers "Cache-Control" => "no-store" if @client_userinfo
34
+ super
35
+ end
36
+
28
37
  def serve_root
29
- cache_control :public, :max_age => 31_536_000
30
- redirect upstream.url(nil, request.query_string)
38
+ cache_control :public, :max_age => 31_536_000 unless upstream.uri.userinfo
39
+ redirect redirect_upstream.url(nil, request.query_string)
31
40
  end
32
41
 
33
42
  def serve_add_gem
@@ -47,15 +56,15 @@ module Gemstash
47
56
  end
48
57
 
49
58
  def serve_dependencies
50
- redirect upstream.url("api/v1/dependencies", request.query_string)
59
+ redirect redirect_upstream.url("api/v1/dependencies", request.query_string)
51
60
  end
52
61
 
53
62
  def serve_dependencies_json
54
- redirect upstream.url("api/v1/dependencies.json", request.query_string)
63
+ redirect redirect_upstream.url("api/v1/dependencies.json", request.query_string)
55
64
  end
56
65
 
57
66
  def serve_names
58
- redirect upstream.url("names", request.query_string)
67
+ redirect redirect_upstream.url("names", request.query_string)
59
68
  end
60
69
 
61
70
  def serve_versions
@@ -63,31 +72,39 @@ module Gemstash
63
72
  end
64
73
 
65
74
  def serve_info(name)
75
+ upstream
76
+ halt 404 unless VALID_ID.match?(name)
66
77
  redirect index_upstream.url("info/#{name}", request.query_string)
67
78
  end
68
79
 
69
80
  def serve_marshal(id)
70
- redirect upstream.url("quick/Marshal.4.8/#{id}", request.query_string)
81
+ upstream
82
+ halt 404 unless valid_file?(id, ".gemspec.rz")
83
+ redirect redirect_upstream.url("quick/Marshal.4.8/#{id}", request.query_string)
71
84
  end
72
85
 
73
86
  def serve_actual_gem(id)
74
- redirect upstream.url("fetch/actual/gem/#{id}", request.query_string)
87
+ upstream
88
+ halt 404 unless valid_file?(id, ".gem")
89
+ redirect redirect_upstream.url("fetch/actual/gem/#{id}", request.query_string)
75
90
  end
76
91
 
77
92
  def serve_gem(id)
78
- redirect upstream.url("gems/#{id}", request.query_string)
93
+ upstream
94
+ halt 404 unless valid_file?(id, ".gem")
95
+ redirect redirect_upstream.url("gems/#{id}", request.query_string)
79
96
  end
80
97
 
81
98
  def serve_latest_specs
82
- redirect upstream.url("latest_specs.4.8.gz", request.query_string)
99
+ redirect redirect_upstream.url("latest_specs.4.8.gz", request.query_string)
83
100
  end
84
101
 
85
102
  def serve_specs
86
- redirect upstream.url("specs.4.8.gz", request.query_string)
103
+ redirect redirect_upstream.url("specs.4.8.gz", request.query_string)
87
104
  end
88
105
 
89
106
  def serve_prerelease_specs
90
- redirect upstream.url("prerelease_specs.4.8.gz", request.query_string)
107
+ redirect redirect_upstream.url("prerelease_specs.4.8.gz", request.query_string)
91
108
  end
92
109
 
93
110
  private
@@ -96,17 +113,71 @@ module Gemstash
96
113
  @upstream ||= Gemstash::Upstream.new(
97
114
  env["gemstash.upstream"],
98
115
  user_agent: env["gemstash.user-agent"]
99
- )
116
+ ).then do |source|
117
+ refuse_upstream(source) unless allowed_upstream?(source)
118
+
119
+ @client_userinfo = source.uri.userinfo unless instance_of?(RubygemsSource) &&
120
+ (gemstash_env.config[:ignore_gemfile_source] || env["HTTP_X_GEMFILE_SOURCE"].to_s.empty?)
121
+ configured = [gemstash_env.config[:rubygems_url], *Array(gemstash_env.config[:allowed_upstreams])]
122
+ match = configured.reject {|entry| entry == "*" }.find {|entry| source.same_source?(Gemstash::Upstream.new(entry)) }
123
+ uri = source.uri.dup
124
+ uri.userinfo ||= Gemstash::Upstream.new(match).uri.userinfo if match
125
+ uri.fragment = nil
126
+ lowercase_host(uri)
127
+ Gemstash::Upstream.new(uri, user_agent: source.user_agent)
128
+ end
129
+ rescue URI::InvalidURIError
130
+ raise Gemstash::WebError.new("Invalid upstream source", 502)
131
+ end
132
+
133
+ # The allowlist ignores the case of a host, so the cache does as well.
134
+ # Newer uri releases drop the userinfo when the host is set.
135
+ def lowercase_host(uri)
136
+ return unless uri.host.to_s.match?(/[A-Z]/)
137
+
138
+ userinfo = uri.userinfo
139
+ uri.host = uri.host.downcase
140
+ uri.userinfo = userinfo
141
+ end
142
+
143
+ def redirect_upstream
144
+ upstream
145
+ return upstream if @client_userinfo
146
+
147
+ uri = upstream.uri.dup
148
+ uri.user = uri.password = nil
149
+ Gemstash::Upstream.new(uri, user_agent: upstream.user_agent)
150
+ end
151
+
152
+ # The default upstream is always allowed, any other one must be listed.
153
+ def allowed_upstream?(upstream)
154
+ upstream.allowed?(gemstash_env.config)
155
+ end
156
+
157
+ def valid_file?(id, suffix)
158
+ VALID_ID.match?(id) && id.end_with?(suffix) && id.length > suffix.length
159
+ end
160
+
161
+ def refuse_upstream(upstream)
162
+ uri = upstream.uri
163
+ source = "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}"
164
+ log.warn "Refusing upstream #{source.inspect}, it is not in :allowed_upstreams"
165
+
166
+ headers "Content-Type" => "text/plain", "Cache-Control" => "no-store"
167
+ halt 403, "This upstream is not allowed by the Gemstash server"
100
168
  end
101
169
 
102
170
  def index_upstream
103
171
  @index_upstream ||=
104
- if upstream.uri.host == "rubygems.org"
105
- uri = upstream.uri.dup
172
+ if redirect_upstream.uri.host == "rubygems.org"
173
+ uri = redirect_upstream.uri.dup
174
+ # Credentials given for rubygems.org must not go to another host.
175
+ # Older uri releases keep userinfo across #host=, so clear it here.
176
+ uri.user = uri.password = nil
106
177
  uri.host = "index.rubygems.org"
107
- Gemstash::Upstream.new(uri, user_agent: upstream.user_agent)
178
+ Gemstash::Upstream.new(uri, user_agent: redirect_upstream.user_agent)
108
179
  else
109
- upstream
180
+ redirect_upstream
110
181
  end
111
182
  end
112
183
  end
@@ -114,7 +185,8 @@ module Gemstash
114
185
  # GemSource for gems in an upstream server.
115
186
  class UpstreamSource < Gemstash::GemSource::RedirectSource
116
187
  include Gemstash::GemSource::DependencyCaching
117
- include Gemstash::Env::Helper
188
+
189
+ VALID_ID = RedirectSource::VALID_ID
118
190
 
119
191
  def self.rack_env_rewriter
120
192
  @rack_env_rewriter ||= Gemstash::RackEnvRewriter.new(%r{\A/upstream/(?<upstream_url>[^/]+)})
@@ -129,35 +201,41 @@ module Gemstash
129
201
  end
130
202
 
131
203
  def serve_latest_specs
132
- http_client = http_client_for(upstream)
133
- http_client.get("latest_specs.4.8.gz")
204
+ serve_index("latest_specs.4.8.gz")
134
205
  end
135
206
 
136
207
  def serve_prerelease_specs
137
- http_client = http_client_for(upstream)
138
- http_client.get("prerelease_specs.4.8.gz")
208
+ serve_index("prerelease_specs.4.8.gz")
139
209
  end
140
210
 
141
211
  def serve_specs
142
- http_client = http_client_for(upstream)
143
- http_client.get("specs.4.8.gz")
212
+ serve_index("specs.4.8.gz")
144
213
  end
145
214
 
146
215
  private
147
216
 
217
+ def serve_index(path)
218
+ index = http_client_for(upstream).get(path)
219
+ content_type "application/octet-stream"
220
+ index
221
+ end
222
+
148
223
  def serve_cached(id, resource_type)
224
+ upstream
225
+ suffix = resource_type == :gem ? ".gem" : ".gemspec.rz"
226
+ halt 404 unless valid_file?(id, suffix)
227
+
149
228
  gem = fetch_gem(id, resource_type)
150
229
  set_gem_headers(gem, resource_type)
151
230
  gem.content(resource_type)
152
- rescue Gemstash::WebError => e
153
- halt e.code
154
231
  end
155
232
 
233
+ # The upstream chose the stored content type, so it is never replayed
156
234
  def set_gem_headers(gem, resource_type)
235
+ content_type "application/octet-stream"
157
236
  return unless gem.property?(:headers, resource_type)
158
237
 
159
238
  gem_headers = gem.properties[:headers][resource_type]
160
- headers["Content-Type"] = gem_headers["content-type"] if gem_headers.include?("content-type")
161
239
  headers["Last-Modified"] = gem_headers["last-modified"] if gem_headers.include?("last-modified")
162
240
  headers["ETag"] = gem_headers["etag"] if gem_headers.include?("etag")
163
241
  end
@@ -169,18 +247,17 @@ module Gemstash
169
247
  end
170
248
  end
171
249
 
172
- def storage
173
- @storage ||= Gemstash::Storage.for("gem_cache")
174
- @storage.for(upstream.host_id)
175
- end
176
-
177
250
  def gem_fetcher
178
251
  @gem_fetcher ||= Gemstash::GemFetcher.new(http_client_for(upstream))
179
252
  end
180
253
 
181
254
  def fetch_gem(id, resource_type)
182
255
  gem_name = Gemstash::Upstream::GemName.new(upstream, id)
183
- gem_resource = storage.resource(gem_name.name)
256
+ storage = Gemstash::Storage.for("gem_cache")
257
+ record = Gemstash::DB::Upstream.find_existing(upstream)
258
+ host_id = record && record.uri == upstream.cache_uri ? record.host_id : upstream.host_id
259
+ gem_resource = storage.for(host_id).resource(gem_name.name)
260
+ gem_resource = configured_cache(gem_name, resource_type, storage) || gem_resource unless gem_resource.exist?(resource_type)
184
261
  if gem_resource.exist?(resource_type)
185
262
  fetch_local_gem(gem_name, gem_resource, resource_type)
186
263
  else
@@ -188,6 +265,24 @@ module Gemstash
188
265
  end
189
266
  end
190
267
 
268
+ def configured_cache(gem_name, resource_type, storage)
269
+ configured = [gemstash_env.config[:rubygems_url], *Array(gemstash_env.config[:allowed_upstreams])]
270
+ configured.each do |entry|
271
+ next if entry == "*"
272
+
273
+ source = Gemstash::Upstream.new(entry)
274
+ next if source.to_s == upstream.to_s || source.uri.userinfo != upstream.uri.userinfo
275
+ next unless source.same_source?(upstream) && source.uri.path.sub(%r{/?\z}, "") == upstream.uri.path.sub(%r{/?\z}, "")
276
+
277
+ old = Gemstash::DB::Upstream[uri: source.to_s] || Gemstash::DB::Upstream[uri: source.cache_uri]
278
+ next unless old && old.host_id == source.host_id
279
+
280
+ resource = storage.for(old.host_id).resource(gem_name.name)
281
+ return resource if resource.exist?(resource_type)
282
+ end
283
+ nil
284
+ end
285
+
191
286
  def fetch_local_gem(gem_name, gem_resource, resource_type)
192
287
  log.info "Gem #{gem_name.name} exists, returning cached #{resource_type}"
193
288
  gem_resource
@@ -197,7 +292,7 @@ module Gemstash
197
292
  log.info "Gem #{gem_name.name} is not cached, fetching #{resource_type}"
198
293
  gem_fetcher.fetch(gem_name.id, resource_type) do |content, properties|
199
294
  resource_properties = {
200
- upstream: upstream.to_s,
295
+ upstream: upstream.cache_uri,
201
296
  gem_name: gem_name.name,
202
297
  headers: { resource_type => properties }
203
298
  }
@@ -3,6 +3,98 @@
3
3
  require "gemstash"
4
4
  require "faraday"
5
5
  require "faraday_middleware"
6
+ require "net/http"
7
+
8
+ # Net::HTTP reads response headers and chunk-size lines into an unbounded
9
+ # buffer before Faraday's on_data callback can enforce the body limit.
10
+ module GemstashHTTPReadLimits
11
+ # Net::BufferedIO's positional boolean is part of its API. Newer net-http
12
+ # also passes a limit: keyword, which is far above Gemstash's own limit.
13
+ # rubocop:disable Style/OptionalBooleanParameter
14
+ def readuntil(terminator, ignore_eof = false, limit: nil)
15
+ limits = Thread.current[:gemstash_http_limits]
16
+ unless limits
17
+ return limit ? super(terminator, ignore_eof, limit: limit) : super(terminator, ignore_eof)
18
+ end
19
+
20
+ offset = @rbuf_offset
21
+ begin
22
+ until (idx = @rbuf.index(terminator, offset))
23
+ raise Gemstash::WebError.new("Upstream response headers are too large", 502) if @rbuf.bytesize - @rbuf_offset > 64 * 1024
24
+
25
+ offset = @rbuf.bytesize
26
+ rbuf_fill
27
+ end
28
+ length = idx + terminator.bytesize - @rbuf_offset
29
+ raise Gemstash::WebError.new("Upstream response headers are too large", 502) if length > 64 * 1024
30
+
31
+ if limits[:headers]
32
+ limits[:metadata] += length
33
+ raise Gemstash::WebError.new("Upstream response headers are too large", 502) if limits[:metadata] > 64 * 1024
34
+ end
35
+
36
+ rbuf_consume(length)
37
+ rescue EOFError
38
+ raise unless ignore_eof
39
+
40
+ rbuf_consume
41
+ end
42
+ end
43
+ # rubocop:enable Style/OptionalBooleanParameter
44
+
45
+ private
46
+
47
+ def rbuf_fill
48
+ limits = Thread.current[:gemstash_http_limits]
49
+ return super unless limits
50
+
51
+ loop do
52
+ now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
53
+ elapsed_after_grace = now - limits[:started] - limits[:grace]
54
+ raise Gemstash::WebError.new("Upstream request timed out", 502) if now >= limits[:deadline] ||
55
+ (elapsed_after_grace > 0 && limits[:bytes] < elapsed_after_grace * 1024)
56
+
57
+ tmp = @rbuf_empty ? @rbuf : nil
58
+ case result = @io.read_nonblock(Net::BufferedIO::BUFSIZE, tmp, exception: false)
59
+ when String
60
+ bytes = result.bytesize
61
+ @rbuf_empty = false
62
+ if result.equal?(tmp)
63
+ @rbuf_offset = 0
64
+ else
65
+ @rbuf << result
66
+ result.clear
67
+ end
68
+ limits[:bytes] += bytes
69
+ return
70
+ when :wait_readable, :wait_writable
71
+ io = @io.to_io
72
+ remaining = [@read_timeout, limits[:deadline] - Process.clock_gettime(Process::CLOCK_MONOTONIC)].min
73
+ raise Gemstash::WebError.new("Upstream request timed out", 502) if remaining <= 0
74
+
75
+ ready = result == :wait_readable ? io.wait_readable(remaining) : io.wait_writable(remaining)
76
+ raise Net::ReadTimeout, "Upstream read timed out" unless ready
77
+ when nil
78
+ raise EOFError, "end of file reached"
79
+ end
80
+ end
81
+ end
82
+ end
83
+
84
+ Net::BufferedIO.prepend(GemstashHTTPReadLimits)
85
+
86
+ # Count only the status and header lines, not chunk-size lines.
87
+ module GemstashHTTPHeaderLimits
88
+ def read_new(sock)
89
+ limits = Thread.current[:gemstash_http_limits]
90
+ limits[:headers] = true if limits
91
+ super
92
+ ensure
93
+ limits[:headers] = false if limits
94
+ end
95
+ end
96
+
97
+ Net::HTTPResponse.singleton_class.prepend(GemstashHTTPHeaderLimits)
6
98
 
7
99
  module Gemstash
8
100
  # :nodoc:
@@ -31,8 +123,10 @@ module Gemstash
31
123
  DEFAULT_USER_AGENT = "Gemstash/#{Gemstash::VERSION}"
32
124
 
33
125
  def self.for(upstream)
34
- client = Faraday.new(upstream.to_s) do |config|
35
- config.use FaradayMiddleware::FollowRedirects
126
+ url = upstream.uri.dup
127
+ url.user = url.password = nil
128
+ client = Faraday.new(url.to_s) do |config|
129
+ config.use FaradayMiddleware::FollowRedirects, callback: method(:refuse_redirect)
36
130
  config.adapter :net_http
37
131
  config.options.timeout = gemstash_env.config[:fetch_timeout]
38
132
  end
@@ -45,6 +139,24 @@ module Gemstash
45
139
  new(client, user_agent: user_agent)
46
140
  end
47
141
 
142
+ def self.refuse_redirect(from, to)
143
+ target = URI(to[:url].to_s)
144
+ if !(from[:url].scheme == "https" && target.scheme != "https") &&
145
+ Gemstash::Upstream.allowed_redirect?(target, gemstash_env.config)
146
+ source = from[:url]
147
+ to[:request_headers].delete("Authorization") if
148
+ [source.scheme, source.hostname.downcase, source.port] != [target.scheme, target.hostname.downcase, target.port]
149
+ return
150
+ end
151
+
152
+ log_target = "#{target.scheme}://#{target.hostname}:#{target.port}#{target.path}"
153
+ Gemstash::Logging.logger.warn("Refusing upstream redirect to #{log_target}")
154
+ raise Gemstash::WebError.new("Refusing an unlisted upstream redirect", 502)
155
+ rescue URI::InvalidURIError, ArgumentError
156
+ Gemstash::Logging.logger.warn("Refusing invalid upstream redirect")
157
+ raise Gemstash::WebError.new("Refusing an invalid upstream redirect", 502)
158
+ end
159
+
48
160
  attr_reader :client
49
161
 
50
162
  def initialize(client = nil, user_agent: nil)
@@ -53,32 +165,79 @@ module Gemstash
53
165
  end
54
166
 
55
167
  def get(path)
168
+ limit = gemstash_env.config[:max_upstream_bytes]
169
+ streamed = nil
170
+
171
+ previous_limits = Thread.current[:gemstash_http_limits]
172
+ started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
173
+ Thread.current[:gemstash_http_limits] = { metadata: 0, headers: false, started: started,
174
+ grace: gemstash_env.config[:fetch_timeout], bytes: 0,
175
+ deadline: started + gemstash_env.config[:max_upstream_duration] }
176
+
56
177
  response = with_retries do
57
178
  @client.get(path) do |req|
58
179
  req.headers["User-Agent"] = @user_agent
59
- req.options.open_timeout = gemstash_env.config[:open_timeout]
180
+ remaining = Thread.current[:gemstash_http_limits][:deadline] - Process.clock_gettime(Process::CLOCK_MONOTONIC)
181
+ raise Gemstash::WebError.new("Upstream request timed out", 502) if remaining <= 0
182
+
183
+ req.options.open_timeout = [gemstash_env.config[:open_timeout], remaining].min
184
+ req.options.timeout = [gemstash_env.config[:fetch_timeout], remaining].min
185
+ streamed = stream_up_to(limit, req) if limit
60
186
  end
61
187
  end
62
188
 
63
- raise Gemstash::WebError.new(response.body, response.status) unless response.success?
189
+ raise Gemstash::WebError.new("Upstream request timed out", 502) if Process.clock_gettime(Process::CLOCK_MONOTONIC) > Thread.current[:gemstash_http_limits][:deadline]
190
+
191
+ # A streamed body is not kept on the response
192
+ body = streamed.to_s.empty? ? response.body : streamed
193
+ raise too_large(limit) if limit && body.to_s.bytesize > limit
194
+ raise Gemstash::WebError.new("Upstream returned HTTP #{response.status}", response.status) unless response.success?
64
195
 
65
196
  if block_given?
66
- yield(response.body, response.headers)
197
+ yield(body, response.headers)
67
198
  else
68
- response.body
199
+ body
69
200
  end
201
+ rescue Faraday::TimeoutError
202
+ raise Gemstash::WebError.new("Upstream request timed out", 502)
203
+ rescue URI::InvalidURIError
204
+ raise Gemstash::WebError.new("Upstream sent an invalid redirect", 502)
205
+ rescue Faraday::Error
206
+ raise Gemstash::WebError.new("Upstream request failed", 502)
207
+ ensure
208
+ Thread.current[:gemstash_http_limits] = previous_limits
70
209
  end
71
210
 
72
211
  private
73
212
 
213
+ # Collects the body as it arrives, so that a download larger than the limit
214
+ # is aborted rather than buffered. Faraday restarts the count of received
215
+ # bytes for each response, which includes any redirect being followed.
216
+ def stream_up_to(limit, req)
217
+ body = String.new(encoding: Encoding::BINARY)
218
+
219
+ req.options.on_data = proc do |chunk, received|
220
+ body.clear if received <= chunk.bytesize
221
+ raise too_large(limit) if received > limit
222
+
223
+ body << chunk
224
+ end
225
+
226
+ body
227
+ end
228
+
229
+ def too_large(limit)
230
+ Gemstash::WebError.new("Upstream response is larger than the #{limit} bytes allowed by :max_upstream_bytes", 502)
231
+ end
232
+
74
233
  def with_retries(times: 3)
75
234
  loop do
76
235
  times -= 1
77
236
  begin
78
237
  return yield
79
238
  rescue Faraday::ConnectionFailed => e
80
- log_error("Connection failure", e)
81
- raise(ConnectionError, e.message) unless times > 0
239
+ log.warn "Upstream connection failure (#{e.class})"
240
+ raise(ConnectionError, "Upstream connection failed") unless times > 0
82
241
 
83
242
  log.info "retrying... #{times} more times"
84
243
  end
@@ -63,7 +63,7 @@ module Gemstash
63
63
 
64
64
  def call(env)
65
65
  env["rack.logger"] = Gemstash::Logging.logger
66
- env["rack.errors"] = Gemstash::Env.current.log_file
66
+ env["rack.errors"] = Gemstash::Logging::StreamLogger.for_stderr
67
67
  @app.call(env)
68
68
  end
69
69
  end
@@ -90,6 +90,8 @@ module Gemstash
90
90
 
91
91
  def flush; end
92
92
 
93
+ def close; end
94
+
93
95
  def sync=(_value); end
94
96
 
95
97
  def sync
@@ -1,73 +1,54 @@
1
- .\" Automatically generated by Pandoc 3.1.3
1
+ .\" Automatically generated by Pandoc 3.7.0.2
2
2
  .\"
3
- .\" Define V font for inline verbatim, using C font in formats
4
- .\" that render this, and otherwise B font.
5
- .ie "\f[CB]x\f[]"x" \{\
6
- . ftr V B
7
- . ftr VI BI
8
- . ftr VB B
9
- . ftr VBI BI
10
- .\}
11
- .el \{\
12
- . ftr V CR
13
- . ftr VI CI
14
- . ftr VB CB
15
- . ftr VBI CBI
16
- .\}
17
- .TH "gemstash-authorize" "1" "October 9, 2015" "" ""
18
- .hy
3
+ .TH "gemstash\-authorize" "1" "October 9, 2015" ""
19
4
  .SH Name
20
- .PP
21
- gemstash-authorize - Adds or removes authorization to interact with
5
+ gemstash\-authorize \- Adds or removes authorization to interact with
22
6
  privately stored gems
23
7
  .SH Synopsis
24
- .PP
25
- \f[V]gemstash authorize [permissions] [--remove] [--list] [--key SECURE_KEY] [--name NAME] [--config-file FILE]\f[R]
8
+ \f[CR]gemstash authorize [permissions] [\-\-remove] [\-\-list] [\-\-key SECURE_KEY] [\-\-name NAME] [\-\-config\-file FILE]\f[R]
26
9
  .SH Description
27
- .PP
28
10
  Adds or removes authorization to interact with privately stored gems.
29
11
  .PP
30
12
  Any arguments will be used as specific permissions.
31
- Valid permissions include \f[V]push\f[R], \f[V]yank\f[R], and
32
- \f[V]fetch\f[R].
13
+ Valid permissions include \f[CR]push\f[R], \f[CR]yank\f[R], and
14
+ \f[CR]fetch\f[R].
33
15
  If no permissions are provided, then all permissions will be granted
34
16
  (including any that may be added in future versions of Gemstash).
35
17
  .SS Usage
36
18
  .IP
37
- .nf
38
- \f[C]
19
+ .EX
39
20
  gemstash authorize
40
21
  gemstash authorize push yank
41
- gemstash authorize push --name my-auth
42
- gemstash authorize yank --key <secure-key>
43
- gemstash authorize --remove --key <secure-key>
44
- gemstash authorize --list
45
- \f[R]
46
- .fi
22
+ gemstash authorize push \-\-name my\-auth
23
+ gemstash authorize yank \-\-key <secure\-key>
24
+ gemstash authorize \-\-remove \-\-key <secure\-key>
25
+ gemstash authorize \-\-list
26
+ .EE
47
27
  .SH Options
48
- .IP \[bu] 2
49
- \f[V]--config-file FILE\f[R]: Specify the config file to use.
50
- If you aren\[cq]t using the default config file at
51
- \f[V]\[ti]/.gemstash/config.yml\f[R] or
52
- \f[V]\[ti]/.gemstash/config.yml.erb\f[R], then you must specify the
28
+ .IP \(bu 2
29
+ \f[CR]\-\-config\-file FILE\f[R]: Specify the config file to use.
30
+ If you aren\(cqt using the default config file at
31
+ \f[CR]\(ti/.gemstash/config.yml\f[R] or
32
+ \f[CR]\(ti/.gemstash/config.yml.erb\f[R], then you must specify the
53
33
  config file via this option.
54
- .IP \[bu] 2
55
- \f[V]--key SECURE_KEY\f[R]: Specify the API key to affect.
34
+ .IP \(bu 2
35
+ \f[CR]\-\-key SECURE_KEY\f[R]: Specify the API key to affect.
56
36
  This should be the actual key value, not a name.
57
- This option is required when using \f[V]--remove\f[R] but is optional
37
+ This option is required when using \f[CR]\-\-remove\f[R] but is optional
58
38
  otherwise.
59
39
  If adding an authorization, using this will either create or update the
60
40
  permissions for the specified API key.
61
41
  If missing, a new API key will always be generated.
62
42
  Note that a key can only have a maximum length of 255 chars.
63
- .IP \[bu] 2
64
- \f[V]--name\f[R]: Name of the authorization.
43
+ .IP \(bu 2
44
+ \f[CR]\-\-name\f[R]: Name of the authorization.
65
45
  Purely for ease of identification, not required.
66
- .IP \[bu] 2
67
- \f[V]--remove\f[R]: Remove an authorization rather than add or update
46
+ .IP \(bu 2
47
+ \f[CR]\-\-remove\f[R]: Remove an authorization rather than add or update
68
48
  one.
69
49
  When removing, permission values are not allowed.
70
- The \f[V]--key <secure-key>\f[R] option is required.
71
- .IP \[bu] 2
72
- \f[V]--list\f[R]: List current authorizations.
73
- Provide \f[V]--name\f[R] or \f[V]--key\f[R] to show only one result.
50
+ The \f[CR]\-\-key <secure\-key>\f[R] option is required.
51
+ .IP \(bu 2
52
+ \f[CR]\-\-list\f[R]: List current authorizations.
53
+ Provide \f[CR]\-\-name\f[R] or \f[CR]\-\-key\f[R] to show only one
54
+ result.