gemstash 2.8.2 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/gemstash/authorization.rb +0 -2
- data/lib/gemstash/cache.rb +5 -4
- data/lib/gemstash/cli/base.rb +9 -2
- data/lib/gemstash/cli/setup.rb +1 -1
- data/lib/gemstash/cli/start.rb +4 -0
- data/lib/gemstash/configuration.rb +71 -1
- data/lib/gemstash/db/upstream.rb +30 -2
- data/lib/gemstash/dependencies.rb +28 -5
- data/lib/gemstash/env.rb +4 -2
- data/lib/gemstash/gem_fetcher.rb +1 -2
- data/lib/gemstash/gem_pusher.rb +24 -20
- data/lib/gemstash/gem_source/private_source.rb +22 -6
- data/lib/gemstash/gem_source/upstream_source.rb +129 -34
- data/lib/gemstash/http_client.rb +167 -8
- data/lib/gemstash/logging.rb +3 -1
- data/lib/gemstash/man/gemstash-authorize.1 +29 -48
- data/lib/gemstash/man/gemstash-authorize.1.txt +14 -14
- data/lib/gemstash/man/gemstash-configuration.5 +231 -154
- data/lib/gemstash/man/gemstash-configuration.5.txt +208 -93
- data/lib/gemstash/man/gemstash-customize.7 +150 -168
- data/lib/gemstash/man/gemstash-customize.7.txt +80 -86
- data/lib/gemstash/man/gemstash-debugging.7 +20 -33
- data/lib/gemstash/man/gemstash-debugging.7.txt +5 -7
- data/lib/gemstash/man/gemstash-deploy.7 +31 -53
- data/lib/gemstash/man/gemstash-deploy.7.txt +17 -18
- data/lib/gemstash/man/gemstash-mirror.7 +24 -37
- data/lib/gemstash/man/gemstash-mirror.7.txt +14 -9
- data/lib/gemstash/man/gemstash-multiple-sources.7 +105 -110
- data/lib/gemstash/man/gemstash-multiple-sources.7.txt +62 -41
- data/lib/gemstash/man/gemstash-private-gems.7 +87 -136
- data/lib/gemstash/man/gemstash-private-gems.7.txt +30 -30
- data/lib/gemstash/man/gemstash-readme.7 +105 -127
- data/lib/gemstash/man/gemstash-readme.7.txt +43 -49
- data/lib/gemstash/man/gemstash-setup.1 +22 -42
- data/lib/gemstash/man/gemstash-setup.1.txt +12 -12
- data/lib/gemstash/man/gemstash-start.1 +9 -27
- data/lib/gemstash/man/gemstash-start.1.txt +5 -5
- data/lib/gemstash/man/gemstash-status.1 +9 -27
- data/lib/gemstash/man/gemstash-status.1.txt +5 -5
- data/lib/gemstash/man/gemstash-stop.1 +9 -27
- data/lib/gemstash/man/gemstash-stop.1.txt +5 -5
- data/lib/gemstash/man/gemstash-version.1 +8 -28
- data/lib/gemstash/man/gemstash-version.1.txt +2 -2
- data/lib/gemstash/rack_env_rewriter.rb +3 -0
- data/lib/gemstash/upstream.rb +83 -6
- data/lib/gemstash/version.rb +1 -1
- data/lib/gemstash/web.rb +12 -0
- metadata +7 -7
|
@@ -7,6 +7,10 @@ module Gemstash
|
|
|
7
7
|
module GemSource
|
|
8
8
|
# GemSource that purely redirects to the upstream server.
|
|
9
9
|
class RedirectSource < Gemstash::GemSource::Base
|
|
10
|
+
include Gemstash::Env::Helper
|
|
11
|
+
|
|
12
|
+
VALID_ID = /\A(?!\.+\z)[A-Za-z0-9._-]+\z/
|
|
13
|
+
|
|
10
14
|
def self.rack_env_rewriter
|
|
11
15
|
@rack_env_rewriter ||= Gemstash::RackEnvRewriter.new(%r{\A/redirect/(?<upstream_url>[^/]+)})
|
|
12
16
|
end
|
|
@@ -16,7 +20,7 @@ module Gemstash
|
|
|
16
20
|
return false unless rewriter.matches?
|
|
17
21
|
|
|
18
22
|
rewriter.rewrite
|
|
19
|
-
env["gemstash.upstream"] = rewriter.captures["upstream_url"]
|
|
23
|
+
env["gemstash.upstream"] = CGI.unescape(rewriter.captures["upstream_url"])
|
|
20
24
|
capture_user_agent(env)
|
|
21
25
|
true
|
|
22
26
|
end
|
|
@@ -25,9 +29,14 @@ module Gemstash
|
|
|
25
29
|
env["gemstash.user-agent"] = env["HTTP_USER_AGENT"]
|
|
26
30
|
end
|
|
27
31
|
|
|
32
|
+
def redirect(target)
|
|
33
|
+
headers "Cache-Control" => "no-store" if @client_userinfo
|
|
34
|
+
super
|
|
35
|
+
end
|
|
36
|
+
|
|
28
37
|
def serve_root
|
|
29
|
-
cache_control :public, :max_age => 31_536_000
|
|
30
|
-
redirect
|
|
38
|
+
cache_control :public, :max_age => 31_536_000 unless upstream.uri.userinfo
|
|
39
|
+
redirect redirect_upstream.url(nil, request.query_string)
|
|
31
40
|
end
|
|
32
41
|
|
|
33
42
|
def serve_add_gem
|
|
@@ -47,15 +56,15 @@ module Gemstash
|
|
|
47
56
|
end
|
|
48
57
|
|
|
49
58
|
def serve_dependencies
|
|
50
|
-
redirect
|
|
59
|
+
redirect redirect_upstream.url("api/v1/dependencies", request.query_string)
|
|
51
60
|
end
|
|
52
61
|
|
|
53
62
|
def serve_dependencies_json
|
|
54
|
-
redirect
|
|
63
|
+
redirect redirect_upstream.url("api/v1/dependencies.json", request.query_string)
|
|
55
64
|
end
|
|
56
65
|
|
|
57
66
|
def serve_names
|
|
58
|
-
redirect
|
|
67
|
+
redirect redirect_upstream.url("names", request.query_string)
|
|
59
68
|
end
|
|
60
69
|
|
|
61
70
|
def serve_versions
|
|
@@ -63,31 +72,39 @@ module Gemstash
|
|
|
63
72
|
end
|
|
64
73
|
|
|
65
74
|
def serve_info(name)
|
|
75
|
+
upstream
|
|
76
|
+
halt 404 unless VALID_ID.match?(name)
|
|
66
77
|
redirect index_upstream.url("info/#{name}", request.query_string)
|
|
67
78
|
end
|
|
68
79
|
|
|
69
80
|
def serve_marshal(id)
|
|
70
|
-
|
|
81
|
+
upstream
|
|
82
|
+
halt 404 unless valid_file?(id, ".gemspec.rz")
|
|
83
|
+
redirect redirect_upstream.url("quick/Marshal.4.8/#{id}", request.query_string)
|
|
71
84
|
end
|
|
72
85
|
|
|
73
86
|
def serve_actual_gem(id)
|
|
74
|
-
|
|
87
|
+
upstream
|
|
88
|
+
halt 404 unless valid_file?(id, ".gem")
|
|
89
|
+
redirect redirect_upstream.url("fetch/actual/gem/#{id}", request.query_string)
|
|
75
90
|
end
|
|
76
91
|
|
|
77
92
|
def serve_gem(id)
|
|
78
|
-
|
|
93
|
+
upstream
|
|
94
|
+
halt 404 unless valid_file?(id, ".gem")
|
|
95
|
+
redirect redirect_upstream.url("gems/#{id}", request.query_string)
|
|
79
96
|
end
|
|
80
97
|
|
|
81
98
|
def serve_latest_specs
|
|
82
|
-
redirect
|
|
99
|
+
redirect redirect_upstream.url("latest_specs.4.8.gz", request.query_string)
|
|
83
100
|
end
|
|
84
101
|
|
|
85
102
|
def serve_specs
|
|
86
|
-
redirect
|
|
103
|
+
redirect redirect_upstream.url("specs.4.8.gz", request.query_string)
|
|
87
104
|
end
|
|
88
105
|
|
|
89
106
|
def serve_prerelease_specs
|
|
90
|
-
redirect
|
|
107
|
+
redirect redirect_upstream.url("prerelease_specs.4.8.gz", request.query_string)
|
|
91
108
|
end
|
|
92
109
|
|
|
93
110
|
private
|
|
@@ -96,17 +113,71 @@ module Gemstash
|
|
|
96
113
|
@upstream ||= Gemstash::Upstream.new(
|
|
97
114
|
env["gemstash.upstream"],
|
|
98
115
|
user_agent: env["gemstash.user-agent"]
|
|
99
|
-
)
|
|
116
|
+
).then do |source|
|
|
117
|
+
refuse_upstream(source) unless allowed_upstream?(source)
|
|
118
|
+
|
|
119
|
+
@client_userinfo = source.uri.userinfo unless instance_of?(RubygemsSource) &&
|
|
120
|
+
(gemstash_env.config[:ignore_gemfile_source] || env["HTTP_X_GEMFILE_SOURCE"].to_s.empty?)
|
|
121
|
+
configured = [gemstash_env.config[:rubygems_url], *Array(gemstash_env.config[:allowed_upstreams])]
|
|
122
|
+
match = configured.reject {|entry| entry == "*" }.find {|entry| source.same_source?(Gemstash::Upstream.new(entry)) }
|
|
123
|
+
uri = source.uri.dup
|
|
124
|
+
uri.userinfo ||= Gemstash::Upstream.new(match).uri.userinfo if match
|
|
125
|
+
uri.fragment = nil
|
|
126
|
+
lowercase_host(uri)
|
|
127
|
+
Gemstash::Upstream.new(uri, user_agent: source.user_agent)
|
|
128
|
+
end
|
|
129
|
+
rescue URI::InvalidURIError
|
|
130
|
+
raise Gemstash::WebError.new("Invalid upstream source", 502)
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# The allowlist ignores the case of a host, so the cache does as well.
|
|
134
|
+
# Newer uri releases drop the userinfo when the host is set.
|
|
135
|
+
def lowercase_host(uri)
|
|
136
|
+
return unless uri.host.to_s.match?(/[A-Z]/)
|
|
137
|
+
|
|
138
|
+
userinfo = uri.userinfo
|
|
139
|
+
uri.host = uri.host.downcase
|
|
140
|
+
uri.userinfo = userinfo
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def redirect_upstream
|
|
144
|
+
upstream
|
|
145
|
+
return upstream if @client_userinfo
|
|
146
|
+
|
|
147
|
+
uri = upstream.uri.dup
|
|
148
|
+
uri.user = uri.password = nil
|
|
149
|
+
Gemstash::Upstream.new(uri, user_agent: upstream.user_agent)
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
# The default upstream is always allowed, any other one must be listed.
|
|
153
|
+
def allowed_upstream?(upstream)
|
|
154
|
+
upstream.allowed?(gemstash_env.config)
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
def valid_file?(id, suffix)
|
|
158
|
+
VALID_ID.match?(id) && id.end_with?(suffix) && id.length > suffix.length
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
def refuse_upstream(upstream)
|
|
162
|
+
uri = upstream.uri
|
|
163
|
+
source = "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}"
|
|
164
|
+
log.warn "Refusing upstream #{source.inspect}, it is not in :allowed_upstreams"
|
|
165
|
+
|
|
166
|
+
headers "Content-Type" => "text/plain", "Cache-Control" => "no-store"
|
|
167
|
+
halt 403, "This upstream is not allowed by the Gemstash server"
|
|
100
168
|
end
|
|
101
169
|
|
|
102
170
|
def index_upstream
|
|
103
171
|
@index_upstream ||=
|
|
104
|
-
if
|
|
105
|
-
uri =
|
|
172
|
+
if redirect_upstream.uri.host == "rubygems.org"
|
|
173
|
+
uri = redirect_upstream.uri.dup
|
|
174
|
+
# Credentials given for rubygems.org must not go to another host.
|
|
175
|
+
# Older uri releases keep userinfo across #host=, so clear it here.
|
|
176
|
+
uri.user = uri.password = nil
|
|
106
177
|
uri.host = "index.rubygems.org"
|
|
107
|
-
Gemstash::Upstream.new(uri, user_agent:
|
|
178
|
+
Gemstash::Upstream.new(uri, user_agent: redirect_upstream.user_agent)
|
|
108
179
|
else
|
|
109
|
-
|
|
180
|
+
redirect_upstream
|
|
110
181
|
end
|
|
111
182
|
end
|
|
112
183
|
end
|
|
@@ -114,7 +185,8 @@ module Gemstash
|
|
|
114
185
|
# GemSource for gems in an upstream server.
|
|
115
186
|
class UpstreamSource < Gemstash::GemSource::RedirectSource
|
|
116
187
|
include Gemstash::GemSource::DependencyCaching
|
|
117
|
-
|
|
188
|
+
|
|
189
|
+
VALID_ID = RedirectSource::VALID_ID
|
|
118
190
|
|
|
119
191
|
def self.rack_env_rewriter
|
|
120
192
|
@rack_env_rewriter ||= Gemstash::RackEnvRewriter.new(%r{\A/upstream/(?<upstream_url>[^/]+)})
|
|
@@ -129,35 +201,41 @@ module Gemstash
|
|
|
129
201
|
end
|
|
130
202
|
|
|
131
203
|
def serve_latest_specs
|
|
132
|
-
|
|
133
|
-
http_client.get("latest_specs.4.8.gz")
|
|
204
|
+
serve_index("latest_specs.4.8.gz")
|
|
134
205
|
end
|
|
135
206
|
|
|
136
207
|
def serve_prerelease_specs
|
|
137
|
-
|
|
138
|
-
http_client.get("prerelease_specs.4.8.gz")
|
|
208
|
+
serve_index("prerelease_specs.4.8.gz")
|
|
139
209
|
end
|
|
140
210
|
|
|
141
211
|
def serve_specs
|
|
142
|
-
|
|
143
|
-
http_client.get("specs.4.8.gz")
|
|
212
|
+
serve_index("specs.4.8.gz")
|
|
144
213
|
end
|
|
145
214
|
|
|
146
215
|
private
|
|
147
216
|
|
|
217
|
+
def serve_index(path)
|
|
218
|
+
index = http_client_for(upstream).get(path)
|
|
219
|
+
content_type "application/octet-stream"
|
|
220
|
+
index
|
|
221
|
+
end
|
|
222
|
+
|
|
148
223
|
def serve_cached(id, resource_type)
|
|
224
|
+
upstream
|
|
225
|
+
suffix = resource_type == :gem ? ".gem" : ".gemspec.rz"
|
|
226
|
+
halt 404 unless valid_file?(id, suffix)
|
|
227
|
+
|
|
149
228
|
gem = fetch_gem(id, resource_type)
|
|
150
229
|
set_gem_headers(gem, resource_type)
|
|
151
230
|
gem.content(resource_type)
|
|
152
|
-
rescue Gemstash::WebError => e
|
|
153
|
-
halt e.code
|
|
154
231
|
end
|
|
155
232
|
|
|
233
|
+
# The upstream chose the stored content type, so it is never replayed
|
|
156
234
|
def set_gem_headers(gem, resource_type)
|
|
235
|
+
content_type "application/octet-stream"
|
|
157
236
|
return unless gem.property?(:headers, resource_type)
|
|
158
237
|
|
|
159
238
|
gem_headers = gem.properties[:headers][resource_type]
|
|
160
|
-
headers["Content-Type"] = gem_headers["content-type"] if gem_headers.include?("content-type")
|
|
161
239
|
headers["Last-Modified"] = gem_headers["last-modified"] if gem_headers.include?("last-modified")
|
|
162
240
|
headers["ETag"] = gem_headers["etag"] if gem_headers.include?("etag")
|
|
163
241
|
end
|
|
@@ -169,18 +247,17 @@ module Gemstash
|
|
|
169
247
|
end
|
|
170
248
|
end
|
|
171
249
|
|
|
172
|
-
def storage
|
|
173
|
-
@storage ||= Gemstash::Storage.for("gem_cache")
|
|
174
|
-
@storage.for(upstream.host_id)
|
|
175
|
-
end
|
|
176
|
-
|
|
177
250
|
def gem_fetcher
|
|
178
251
|
@gem_fetcher ||= Gemstash::GemFetcher.new(http_client_for(upstream))
|
|
179
252
|
end
|
|
180
253
|
|
|
181
254
|
def fetch_gem(id, resource_type)
|
|
182
255
|
gem_name = Gemstash::Upstream::GemName.new(upstream, id)
|
|
183
|
-
|
|
256
|
+
storage = Gemstash::Storage.for("gem_cache")
|
|
257
|
+
record = Gemstash::DB::Upstream.find_existing(upstream)
|
|
258
|
+
host_id = record && record.uri == upstream.cache_uri ? record.host_id : upstream.host_id
|
|
259
|
+
gem_resource = storage.for(host_id).resource(gem_name.name)
|
|
260
|
+
gem_resource = configured_cache(gem_name, resource_type, storage) || gem_resource unless gem_resource.exist?(resource_type)
|
|
184
261
|
if gem_resource.exist?(resource_type)
|
|
185
262
|
fetch_local_gem(gem_name, gem_resource, resource_type)
|
|
186
263
|
else
|
|
@@ -188,6 +265,24 @@ module Gemstash
|
|
|
188
265
|
end
|
|
189
266
|
end
|
|
190
267
|
|
|
268
|
+
def configured_cache(gem_name, resource_type, storage)
|
|
269
|
+
configured = [gemstash_env.config[:rubygems_url], *Array(gemstash_env.config[:allowed_upstreams])]
|
|
270
|
+
configured.each do |entry|
|
|
271
|
+
next if entry == "*"
|
|
272
|
+
|
|
273
|
+
source = Gemstash::Upstream.new(entry)
|
|
274
|
+
next if source.to_s == upstream.to_s || source.uri.userinfo != upstream.uri.userinfo
|
|
275
|
+
next unless source.same_source?(upstream) && source.uri.path.sub(%r{/?\z}, "") == upstream.uri.path.sub(%r{/?\z}, "")
|
|
276
|
+
|
|
277
|
+
old = Gemstash::DB::Upstream[uri: source.to_s] || Gemstash::DB::Upstream[uri: source.cache_uri]
|
|
278
|
+
next unless old && old.host_id == source.host_id
|
|
279
|
+
|
|
280
|
+
resource = storage.for(old.host_id).resource(gem_name.name)
|
|
281
|
+
return resource if resource.exist?(resource_type)
|
|
282
|
+
end
|
|
283
|
+
nil
|
|
284
|
+
end
|
|
285
|
+
|
|
191
286
|
def fetch_local_gem(gem_name, gem_resource, resource_type)
|
|
192
287
|
log.info "Gem #{gem_name.name} exists, returning cached #{resource_type}"
|
|
193
288
|
gem_resource
|
|
@@ -197,7 +292,7 @@ module Gemstash
|
|
|
197
292
|
log.info "Gem #{gem_name.name} is not cached, fetching #{resource_type}"
|
|
198
293
|
gem_fetcher.fetch(gem_name.id, resource_type) do |content, properties|
|
|
199
294
|
resource_properties = {
|
|
200
|
-
upstream: upstream.
|
|
295
|
+
upstream: upstream.cache_uri,
|
|
201
296
|
gem_name: gem_name.name,
|
|
202
297
|
headers: { resource_type => properties }
|
|
203
298
|
}
|
data/lib/gemstash/http_client.rb
CHANGED
|
@@ -3,6 +3,98 @@
|
|
|
3
3
|
require "gemstash"
|
|
4
4
|
require "faraday"
|
|
5
5
|
require "faraday_middleware"
|
|
6
|
+
require "net/http"
|
|
7
|
+
|
|
8
|
+
# Net::HTTP reads response headers and chunk-size lines into an unbounded
|
|
9
|
+
# buffer before Faraday's on_data callback can enforce the body limit.
|
|
10
|
+
module GemstashHTTPReadLimits
|
|
11
|
+
# Net::BufferedIO's positional boolean is part of its API. Newer net-http
|
|
12
|
+
# also passes a limit: keyword, which is far above Gemstash's own limit.
|
|
13
|
+
# rubocop:disable Style/OptionalBooleanParameter
|
|
14
|
+
def readuntil(terminator, ignore_eof = false, limit: nil)
|
|
15
|
+
limits = Thread.current[:gemstash_http_limits]
|
|
16
|
+
unless limits
|
|
17
|
+
return limit ? super(terminator, ignore_eof, limit: limit) : super(terminator, ignore_eof)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
offset = @rbuf_offset
|
|
21
|
+
begin
|
|
22
|
+
until (idx = @rbuf.index(terminator, offset))
|
|
23
|
+
raise Gemstash::WebError.new("Upstream response headers are too large", 502) if @rbuf.bytesize - @rbuf_offset > 64 * 1024
|
|
24
|
+
|
|
25
|
+
offset = @rbuf.bytesize
|
|
26
|
+
rbuf_fill
|
|
27
|
+
end
|
|
28
|
+
length = idx + terminator.bytesize - @rbuf_offset
|
|
29
|
+
raise Gemstash::WebError.new("Upstream response headers are too large", 502) if length > 64 * 1024
|
|
30
|
+
|
|
31
|
+
if limits[:headers]
|
|
32
|
+
limits[:metadata] += length
|
|
33
|
+
raise Gemstash::WebError.new("Upstream response headers are too large", 502) if limits[:metadata] > 64 * 1024
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
rbuf_consume(length)
|
|
37
|
+
rescue EOFError
|
|
38
|
+
raise unless ignore_eof
|
|
39
|
+
|
|
40
|
+
rbuf_consume
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
# rubocop:enable Style/OptionalBooleanParameter
|
|
44
|
+
|
|
45
|
+
private
|
|
46
|
+
|
|
47
|
+
def rbuf_fill
|
|
48
|
+
limits = Thread.current[:gemstash_http_limits]
|
|
49
|
+
return super unless limits
|
|
50
|
+
|
|
51
|
+
loop do
|
|
52
|
+
now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
53
|
+
elapsed_after_grace = now - limits[:started] - limits[:grace]
|
|
54
|
+
raise Gemstash::WebError.new("Upstream request timed out", 502) if now >= limits[:deadline] ||
|
|
55
|
+
(elapsed_after_grace > 0 && limits[:bytes] < elapsed_after_grace * 1024)
|
|
56
|
+
|
|
57
|
+
tmp = @rbuf_empty ? @rbuf : nil
|
|
58
|
+
case result = @io.read_nonblock(Net::BufferedIO::BUFSIZE, tmp, exception: false)
|
|
59
|
+
when String
|
|
60
|
+
bytes = result.bytesize
|
|
61
|
+
@rbuf_empty = false
|
|
62
|
+
if result.equal?(tmp)
|
|
63
|
+
@rbuf_offset = 0
|
|
64
|
+
else
|
|
65
|
+
@rbuf << result
|
|
66
|
+
result.clear
|
|
67
|
+
end
|
|
68
|
+
limits[:bytes] += bytes
|
|
69
|
+
return
|
|
70
|
+
when :wait_readable, :wait_writable
|
|
71
|
+
io = @io.to_io
|
|
72
|
+
remaining = [@read_timeout, limits[:deadline] - Process.clock_gettime(Process::CLOCK_MONOTONIC)].min
|
|
73
|
+
raise Gemstash::WebError.new("Upstream request timed out", 502) if remaining <= 0
|
|
74
|
+
|
|
75
|
+
ready = result == :wait_readable ? io.wait_readable(remaining) : io.wait_writable(remaining)
|
|
76
|
+
raise Net::ReadTimeout, "Upstream read timed out" unless ready
|
|
77
|
+
when nil
|
|
78
|
+
raise EOFError, "end of file reached"
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
Net::BufferedIO.prepend(GemstashHTTPReadLimits)
|
|
85
|
+
|
|
86
|
+
# Count only the status and header lines, not chunk-size lines.
|
|
87
|
+
module GemstashHTTPHeaderLimits
|
|
88
|
+
def read_new(sock)
|
|
89
|
+
limits = Thread.current[:gemstash_http_limits]
|
|
90
|
+
limits[:headers] = true if limits
|
|
91
|
+
super
|
|
92
|
+
ensure
|
|
93
|
+
limits[:headers] = false if limits
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
Net::HTTPResponse.singleton_class.prepend(GemstashHTTPHeaderLimits)
|
|
6
98
|
|
|
7
99
|
module Gemstash
|
|
8
100
|
# :nodoc:
|
|
@@ -31,8 +123,10 @@ module Gemstash
|
|
|
31
123
|
DEFAULT_USER_AGENT = "Gemstash/#{Gemstash::VERSION}"
|
|
32
124
|
|
|
33
125
|
def self.for(upstream)
|
|
34
|
-
|
|
35
|
-
|
|
126
|
+
url = upstream.uri.dup
|
|
127
|
+
url.user = url.password = nil
|
|
128
|
+
client = Faraday.new(url.to_s) do |config|
|
|
129
|
+
config.use FaradayMiddleware::FollowRedirects, callback: method(:refuse_redirect)
|
|
36
130
|
config.adapter :net_http
|
|
37
131
|
config.options.timeout = gemstash_env.config[:fetch_timeout]
|
|
38
132
|
end
|
|
@@ -45,6 +139,24 @@ module Gemstash
|
|
|
45
139
|
new(client, user_agent: user_agent)
|
|
46
140
|
end
|
|
47
141
|
|
|
142
|
+
def self.refuse_redirect(from, to)
|
|
143
|
+
target = URI(to[:url].to_s)
|
|
144
|
+
if !(from[:url].scheme == "https" && target.scheme != "https") &&
|
|
145
|
+
Gemstash::Upstream.allowed_redirect?(target, gemstash_env.config)
|
|
146
|
+
source = from[:url]
|
|
147
|
+
to[:request_headers].delete("Authorization") if
|
|
148
|
+
[source.scheme, source.hostname.downcase, source.port] != [target.scheme, target.hostname.downcase, target.port]
|
|
149
|
+
return
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
log_target = "#{target.scheme}://#{target.hostname}:#{target.port}#{target.path}"
|
|
153
|
+
Gemstash::Logging.logger.warn("Refusing upstream redirect to #{log_target}")
|
|
154
|
+
raise Gemstash::WebError.new("Refusing an unlisted upstream redirect", 502)
|
|
155
|
+
rescue URI::InvalidURIError, ArgumentError
|
|
156
|
+
Gemstash::Logging.logger.warn("Refusing invalid upstream redirect")
|
|
157
|
+
raise Gemstash::WebError.new("Refusing an invalid upstream redirect", 502)
|
|
158
|
+
end
|
|
159
|
+
|
|
48
160
|
attr_reader :client
|
|
49
161
|
|
|
50
162
|
def initialize(client = nil, user_agent: nil)
|
|
@@ -53,32 +165,79 @@ module Gemstash
|
|
|
53
165
|
end
|
|
54
166
|
|
|
55
167
|
def get(path)
|
|
168
|
+
limit = gemstash_env.config[:max_upstream_bytes]
|
|
169
|
+
streamed = nil
|
|
170
|
+
|
|
171
|
+
previous_limits = Thread.current[:gemstash_http_limits]
|
|
172
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
173
|
+
Thread.current[:gemstash_http_limits] = { metadata: 0, headers: false, started: started,
|
|
174
|
+
grace: gemstash_env.config[:fetch_timeout], bytes: 0,
|
|
175
|
+
deadline: started + gemstash_env.config[:max_upstream_duration] }
|
|
176
|
+
|
|
56
177
|
response = with_retries do
|
|
57
178
|
@client.get(path) do |req|
|
|
58
179
|
req.headers["User-Agent"] = @user_agent
|
|
59
|
-
|
|
180
|
+
remaining = Thread.current[:gemstash_http_limits][:deadline] - Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
181
|
+
raise Gemstash::WebError.new("Upstream request timed out", 502) if remaining <= 0
|
|
182
|
+
|
|
183
|
+
req.options.open_timeout = [gemstash_env.config[:open_timeout], remaining].min
|
|
184
|
+
req.options.timeout = [gemstash_env.config[:fetch_timeout], remaining].min
|
|
185
|
+
streamed = stream_up_to(limit, req) if limit
|
|
60
186
|
end
|
|
61
187
|
end
|
|
62
188
|
|
|
63
|
-
raise Gemstash::WebError.new(
|
|
189
|
+
raise Gemstash::WebError.new("Upstream request timed out", 502) if Process.clock_gettime(Process::CLOCK_MONOTONIC) > Thread.current[:gemstash_http_limits][:deadline]
|
|
190
|
+
|
|
191
|
+
# A streamed body is not kept on the response
|
|
192
|
+
body = streamed.to_s.empty? ? response.body : streamed
|
|
193
|
+
raise too_large(limit) if limit && body.to_s.bytesize > limit
|
|
194
|
+
raise Gemstash::WebError.new("Upstream returned HTTP #{response.status}", response.status) unless response.success?
|
|
64
195
|
|
|
65
196
|
if block_given?
|
|
66
|
-
yield(
|
|
197
|
+
yield(body, response.headers)
|
|
67
198
|
else
|
|
68
|
-
|
|
199
|
+
body
|
|
69
200
|
end
|
|
201
|
+
rescue Faraday::TimeoutError
|
|
202
|
+
raise Gemstash::WebError.new("Upstream request timed out", 502)
|
|
203
|
+
rescue URI::InvalidURIError
|
|
204
|
+
raise Gemstash::WebError.new("Upstream sent an invalid redirect", 502)
|
|
205
|
+
rescue Faraday::Error
|
|
206
|
+
raise Gemstash::WebError.new("Upstream request failed", 502)
|
|
207
|
+
ensure
|
|
208
|
+
Thread.current[:gemstash_http_limits] = previous_limits
|
|
70
209
|
end
|
|
71
210
|
|
|
72
211
|
private
|
|
73
212
|
|
|
213
|
+
# Collects the body as it arrives, so that a download larger than the limit
|
|
214
|
+
# is aborted rather than buffered. Faraday restarts the count of received
|
|
215
|
+
# bytes for each response, which includes any redirect being followed.
|
|
216
|
+
def stream_up_to(limit, req)
|
|
217
|
+
body = String.new(encoding: Encoding::BINARY)
|
|
218
|
+
|
|
219
|
+
req.options.on_data = proc do |chunk, received|
|
|
220
|
+
body.clear if received <= chunk.bytesize
|
|
221
|
+
raise too_large(limit) if received > limit
|
|
222
|
+
|
|
223
|
+
body << chunk
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
body
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
def too_large(limit)
|
|
230
|
+
Gemstash::WebError.new("Upstream response is larger than the #{limit} bytes allowed by :max_upstream_bytes", 502)
|
|
231
|
+
end
|
|
232
|
+
|
|
74
233
|
def with_retries(times: 3)
|
|
75
234
|
loop do
|
|
76
235
|
times -= 1
|
|
77
236
|
begin
|
|
78
237
|
return yield
|
|
79
238
|
rescue Faraday::ConnectionFailed => e
|
|
80
|
-
|
|
81
|
-
raise(ConnectionError,
|
|
239
|
+
log.warn "Upstream connection failure (#{e.class})"
|
|
240
|
+
raise(ConnectionError, "Upstream connection failed") unless times > 0
|
|
82
241
|
|
|
83
242
|
log.info "retrying... #{times} more times"
|
|
84
243
|
end
|
data/lib/gemstash/logging.rb
CHANGED
|
@@ -63,7 +63,7 @@ module Gemstash
|
|
|
63
63
|
|
|
64
64
|
def call(env)
|
|
65
65
|
env["rack.logger"] = Gemstash::Logging.logger
|
|
66
|
-
env["rack.errors"] = Gemstash::
|
|
66
|
+
env["rack.errors"] = Gemstash::Logging::StreamLogger.for_stderr
|
|
67
67
|
@app.call(env)
|
|
68
68
|
end
|
|
69
69
|
end
|
|
@@ -90,6 +90,8 @@ module Gemstash
|
|
|
90
90
|
|
|
91
91
|
def flush; end
|
|
92
92
|
|
|
93
|
+
def close; end
|
|
94
|
+
|
|
93
95
|
def sync=(_value); end
|
|
94
96
|
|
|
95
97
|
def sync
|
|
@@ -1,73 +1,54 @@
|
|
|
1
|
-
.\" Automatically generated by Pandoc 3.
|
|
1
|
+
.\" Automatically generated by Pandoc 3.7.0.2
|
|
2
2
|
.\"
|
|
3
|
-
|
|
4
|
-
.\" that render this, and otherwise B font.
|
|
5
|
-
.ie "\f[CB]x\f[]"x" \{\
|
|
6
|
-
. ftr V B
|
|
7
|
-
. ftr VI BI
|
|
8
|
-
. ftr VB B
|
|
9
|
-
. ftr VBI BI
|
|
10
|
-
.\}
|
|
11
|
-
.el \{\
|
|
12
|
-
. ftr V CR
|
|
13
|
-
. ftr VI CI
|
|
14
|
-
. ftr VB CB
|
|
15
|
-
. ftr VBI CBI
|
|
16
|
-
.\}
|
|
17
|
-
.TH "gemstash-authorize" "1" "October 9, 2015" "" ""
|
|
18
|
-
.hy
|
|
3
|
+
.TH "gemstash\-authorize" "1" "October 9, 2015" ""
|
|
19
4
|
.SH Name
|
|
20
|
-
|
|
21
|
-
gemstash-authorize - Adds or removes authorization to interact with
|
|
5
|
+
gemstash\-authorize \- Adds or removes authorization to interact with
|
|
22
6
|
privately stored gems
|
|
23
7
|
.SH Synopsis
|
|
24
|
-
|
|
25
|
-
\f[V]gemstash authorize [permissions] [--remove] [--list] [--key SECURE_KEY] [--name NAME] [--config-file FILE]\f[R]
|
|
8
|
+
\f[CR]gemstash authorize [permissions] [\-\-remove] [\-\-list] [\-\-key SECURE_KEY] [\-\-name NAME] [\-\-config\-file FILE]\f[R]
|
|
26
9
|
.SH Description
|
|
27
|
-
.PP
|
|
28
10
|
Adds or removes authorization to interact with privately stored gems.
|
|
29
11
|
.PP
|
|
30
12
|
Any arguments will be used as specific permissions.
|
|
31
|
-
Valid permissions include \f[
|
|
32
|
-
\f[
|
|
13
|
+
Valid permissions include \f[CR]push\f[R], \f[CR]yank\f[R], and
|
|
14
|
+
\f[CR]fetch\f[R].
|
|
33
15
|
If no permissions are provided, then all permissions will be granted
|
|
34
16
|
(including any that may be added in future versions of Gemstash).
|
|
35
17
|
.SS Usage
|
|
36
18
|
.IP
|
|
37
|
-
.
|
|
38
|
-
\f[C]
|
|
19
|
+
.EX
|
|
39
20
|
gemstash authorize
|
|
40
21
|
gemstash authorize push yank
|
|
41
|
-
gemstash authorize push
|
|
42
|
-
gemstash authorize yank
|
|
43
|
-
gemstash authorize
|
|
44
|
-
gemstash authorize
|
|
45
|
-
|
|
46
|
-
.fi
|
|
22
|
+
gemstash authorize push \-\-name my\-auth
|
|
23
|
+
gemstash authorize yank \-\-key <secure\-key>
|
|
24
|
+
gemstash authorize \-\-remove \-\-key <secure\-key>
|
|
25
|
+
gemstash authorize \-\-list
|
|
26
|
+
.EE
|
|
47
27
|
.SH Options
|
|
48
|
-
.IP \
|
|
49
|
-
\f[
|
|
50
|
-
If you aren\
|
|
51
|
-
\f[
|
|
52
|
-
\f[
|
|
28
|
+
.IP \(bu 2
|
|
29
|
+
\f[CR]\-\-config\-file FILE\f[R]: Specify the config file to use.
|
|
30
|
+
If you aren\(cqt using the default config file at
|
|
31
|
+
\f[CR]\(ti/.gemstash/config.yml\f[R] or
|
|
32
|
+
\f[CR]\(ti/.gemstash/config.yml.erb\f[R], then you must specify the
|
|
53
33
|
config file via this option.
|
|
54
|
-
.IP \
|
|
55
|
-
\f[
|
|
34
|
+
.IP \(bu 2
|
|
35
|
+
\f[CR]\-\-key SECURE_KEY\f[R]: Specify the API key to affect.
|
|
56
36
|
This should be the actual key value, not a name.
|
|
57
|
-
This option is required when using \f[
|
|
37
|
+
This option is required when using \f[CR]\-\-remove\f[R] but is optional
|
|
58
38
|
otherwise.
|
|
59
39
|
If adding an authorization, using this will either create or update the
|
|
60
40
|
permissions for the specified API key.
|
|
61
41
|
If missing, a new API key will always be generated.
|
|
62
42
|
Note that a key can only have a maximum length of 255 chars.
|
|
63
|
-
.IP \
|
|
64
|
-
\f[
|
|
43
|
+
.IP \(bu 2
|
|
44
|
+
\f[CR]\-\-name\f[R]: Name of the authorization.
|
|
65
45
|
Purely for ease of identification, not required.
|
|
66
|
-
.IP \
|
|
67
|
-
\f[
|
|
46
|
+
.IP \(bu 2
|
|
47
|
+
\f[CR]\-\-remove\f[R]: Remove an authorization rather than add or update
|
|
68
48
|
one.
|
|
69
49
|
When removing, permission values are not allowed.
|
|
70
|
-
The \f[
|
|
71
|
-
.IP \
|
|
72
|
-
\f[
|
|
73
|
-
Provide \f[
|
|
50
|
+
The \f[CR]\-\-key <secure\-key>\f[R] option is required.
|
|
51
|
+
.IP \(bu 2
|
|
52
|
+
\f[CR]\-\-list\f[R]: List current authorizations.
|
|
53
|
+
Provide \f[CR]\-\-name\f[R] or \f[CR]\-\-key\f[R] to show only one
|
|
54
|
+
result.
|