constile_gate 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +226 -0
- data/constile_gate.gemspec +21 -0
- data/lib/constile_gate/agents.rb +50 -0
- data/lib/constile_gate/charset.rb +427 -0
- data/lib/constile_gate/charset_tables.rb +284 -0
- data/lib/constile_gate/gate.rb +612 -0
- data/lib/constile_gate/http.rb +235 -0
- data/lib/constile_gate/js.rb +99 -0
- data/lib/constile_gate/page_meta.rb +83 -0
- data/lib/constile_gate/rack.rb +371 -0
- data/lib/constile_gate/railtie.rb +17 -0
- data/lib/constile_gate/url.rb +208 -0
- data/lib/constile_gate/version.rb +5 -0
- data/lib/constile_gate.rb +6 -0
- metadata +62 -0
|
@@ -0,0 +1,612 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
require "constile_gate/version"
|
|
3
|
+
require "constile_gate/js"
|
|
4
|
+
require "constile_gate/url"
|
|
5
|
+
require "constile_gate/page_meta"
|
|
6
|
+
require "constile_gate/agents"
|
|
7
|
+
require "constile_gate/charset"
|
|
8
|
+
require "constile_gate/http"
|
|
9
|
+
|
|
10
|
+
# The publisher's gate, ported from packages/gate/src/core.ts. It sits in front of
|
|
11
|
+
# the publisher's site and does three things:
|
|
12
|
+
#
|
|
13
|
+
# 1. For an agent fetching an eligible page for a person, it serves the
|
|
14
|
+
# Constile offer page with HTTP 200 instead of the article.
|
|
15
|
+
# 2. It forwards the offer's callbacks (/sponsored/...) to the network and, when
|
|
16
|
+
# the network grants access, returns the article from the origin.
|
|
17
|
+
# 3. Everything else (people, search engines, crawlers, subscribers, other paths)
|
|
18
|
+
# goes straight to the origin, untouched.
|
|
19
|
+
#
|
|
20
|
+
# It fails open: if the network can't be reached, the page is served as if the
|
|
21
|
+
# gate weren't there. Callbacks can't fail open, because they release content
|
|
22
|
+
# only against a recorded commitment.
|
|
23
|
+
module ConstileGate
|
|
24
|
+
# What the gate needs for one request.
|
|
25
|
+
# site_id, secret, network_url the site's settings; network_url is the network's origin
|
|
26
|
+
# origin callable(Request) -> Response: fetches from the publisher's site
|
|
27
|
+
# page_origin callable(Request) -> Response: fetches the page an offer is for; defaults to origin.
|
|
28
|
+
# An in-process adapter uses it to ask the app for the page uncompressed.
|
|
29
|
+
# network callable(NetworkRequest) -> NetworkResponse; defaults to HTTPClient
|
|
30
|
+
# signals { asn:, as_org:, country:, ip_prefix:, ip: }; ip is sent only for registered agents
|
|
31
|
+
# now callable -> seconds since the epoch
|
|
32
|
+
# config_retry_s after a failed config fetch, seconds to pass everything through before asking again (default 30)
|
|
33
|
+
# wait_until callable(job): runs job (a callable) after the response; without it, the job runs first
|
|
34
|
+
# platform where the gate runs, such as "rack" or "rails"
|
|
35
|
+
# logger callable(message) or an object with #warn; defaults to Kernel#warn
|
|
36
|
+
Deps = Struct.new(:site_id, :secret, :network_url, :origin, :page_origin, :network, :signals, :now, :config_retry_s,
|
|
37
|
+
:wait_until, :platform, :logger, keyword_init: true) do
|
|
38
|
+
# The secret never prints: not in logs, error pages, or pp.
|
|
39
|
+
def inspect
|
|
40
|
+
fields = members.map { |m| "#{m}=#{m == :secret && !self[m].nil? ? "[REDACTED]" : self[m].inspect}" }
|
|
41
|
+
"#<struct #{self.class.name} #{fields.join(", ")}>"
|
|
42
|
+
end
|
|
43
|
+
alias_method :to_s, :inspect
|
|
44
|
+
|
|
45
|
+
def pretty_print(q)
|
|
46
|
+
q.text(inspect)
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# The network answered with an error status. The message is the network's error.
|
|
51
|
+
class NetworkError < StandardError
|
|
52
|
+
attr_reader :status
|
|
53
|
+
|
|
54
|
+
def initialize(status, message)
|
|
55
|
+
super(message)
|
|
56
|
+
@status = status
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
SPONSORED_HEADERS = { "cache-control" => "no-store", "x-robots-tag" => "noindex, nofollow", "vary" => "User-Agent" }.freeze
|
|
61
|
+
CALLBACK = %r{\A/sponsored/(deferred|prepared|delivered)/([A-Za-z0-9_-]+)\z}.freeze
|
|
62
|
+
CONTENT = %r{\A/sponsored/content/([A-Za-z0-9_-]+)\z}.freeze
|
|
63
|
+
MACHINE_OFFER = %r{\A/sponsored/offer/([A-Za-z0-9_-]+)\z}.freeze
|
|
64
|
+
HTML_TYPE = /#{PageMeta.ci("text/html")}|#{PageMeta.ci("application/xhtml")}/.freeze
|
|
65
|
+
RELEASE_ACCEPT = "text/html,application/xhtml+xml,text/plain;q=0.9,*/*;q=0.5".freeze
|
|
66
|
+
UNAVAILABLE = "Sponsored access is unavailable right now".freeze
|
|
67
|
+
|
|
68
|
+
CONFIG_RETRY_S = 30
|
|
69
|
+
# How long a config may be used past its TTL while the network can't give a fresh one.
|
|
70
|
+
CONFIG_STALE_S = 86_400
|
|
71
|
+
# How long a request waits for a fetch another request started, at most. A fetch or refresh
|
|
72
|
+
# older than this (or one a parent process started before forking) is treated as gone.
|
|
73
|
+
CONFIG_WAIT_S = 30
|
|
74
|
+
|
|
75
|
+
# Configs from the network, per network and site, kept for their ttl_s (10 to 3,600 seconds),
|
|
76
|
+
# and when the network last failed to give one: the gate leaves it alone until then, so
|
|
77
|
+
# people aren't held up. Also the fetches in flight: requests with no config share one, and
|
|
78
|
+
# one background refresh at a time keeps a stale config fresh.
|
|
79
|
+
module ConfigCache
|
|
80
|
+
# A fetch for requests with no config: the first request fetches, the others wait for its result.
|
|
81
|
+
Fetch = Struct.new(:started, :pid, :done, :config, :error, :cond)
|
|
82
|
+
|
|
83
|
+
@entries = {}
|
|
84
|
+
@failures = {}
|
|
85
|
+
@fetches = {}
|
|
86
|
+
@refreshing = {}
|
|
87
|
+
@mutex = Mutex.new
|
|
88
|
+
|
|
89
|
+
def self.clock
|
|
90
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def self.live?(started, pid)
|
|
94
|
+
pid == Process.pid && clock - started <= CONFIG_WAIT_S
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def self.get(key)
|
|
98
|
+
@mutex.synchronize { @entries[key] }
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def self.set(key, config, until_s)
|
|
102
|
+
@mutex.synchronize { @entries[key] = { config: config, until: until_s } }
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def self.failed_until(key)
|
|
106
|
+
@mutex.synchronize { @failures[key] }
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def self.fail(key, until_s)
|
|
110
|
+
@mutex.synchronize { @failures[key] = until_s }
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def self.recover(key)
|
|
114
|
+
@mutex.synchronize { @failures.delete(key) }
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
# [fetch, owner]: the fetch in flight for key, or a new one this request owns.
|
|
118
|
+
def self.claim_fetch(key)
|
|
119
|
+
@mutex.synchronize do
|
|
120
|
+
fetch = @fetches[key]
|
|
121
|
+
return [fetch, false] if fetch && live?(fetch.started, fetch.pid)
|
|
122
|
+
|
|
123
|
+
fetch = Fetch.new(clock, Process.pid, false, nil, nil, ConditionVariable.new)
|
|
124
|
+
@fetches[key] = fetch
|
|
125
|
+
[fetch, true]
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def self.finish_fetch(key, fetch, config, error)
|
|
130
|
+
@mutex.synchronize do
|
|
131
|
+
fetch.config = config
|
|
132
|
+
fetch.error = error
|
|
133
|
+
fetch.done = true
|
|
134
|
+
@fetches.delete(key) if @fetches[key].equal?(fetch)
|
|
135
|
+
fetch.cond.broadcast
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
# The shared fetch's config, or its error raised here.
|
|
140
|
+
def self.wait_fetch(fetch)
|
|
141
|
+
deadline = clock + CONFIG_WAIT_S
|
|
142
|
+
@mutex.synchronize do
|
|
143
|
+
until fetch.done
|
|
144
|
+
left = deadline - clock
|
|
145
|
+
raise "the shared config fetch didn't finish within #{CONFIG_WAIT_S} s" if left <= 0
|
|
146
|
+
|
|
147
|
+
fetch.cond.wait(@mutex, left)
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
raise fetch.error if fetch.error
|
|
151
|
+
|
|
152
|
+
fetch.config
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
# True when this request should start the refresh: none is running for key.
|
|
156
|
+
def self.start_refresh(key)
|
|
157
|
+
@mutex.synchronize do
|
|
158
|
+
started = @refreshing[key]
|
|
159
|
+
return false if started && live?(*started)
|
|
160
|
+
|
|
161
|
+
@refreshing[key] = [clock, Process.pid]
|
|
162
|
+
true
|
|
163
|
+
end
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def self.end_refresh(key)
|
|
167
|
+
@mutex.synchronize { @refreshing.delete(key) }
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
def self.clear
|
|
171
|
+
@mutex.synchronize do
|
|
172
|
+
@entries.clear
|
|
173
|
+
@failures.clear
|
|
174
|
+
@fetches.clear
|
|
175
|
+
@refreshing.clear
|
|
176
|
+
end
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
module_function
|
|
181
|
+
|
|
182
|
+
# For tests: forget cached configs and failures.
|
|
183
|
+
def clear_gate_cache
|
|
184
|
+
ConfigCache.clear
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
# Whether the gate itself answers this path: the seller file and the /sponsored/ routes.
|
|
188
|
+
def gate_route?(path)
|
|
189
|
+
path == "/.well-known/constile" || CALLBACK.match?(path) || CONTENT.match?(path) || MACHINE_OFFER.match?(path)
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
# The first three parts of an address, as in the research gateway's request log.
|
|
193
|
+
# CONSTILE_DISABLED's on values: 1, true, yes, or on, in any case.
|
|
194
|
+
def on?(value)
|
|
195
|
+
/\A(1|true|yes|on)\z/i.match?(value.to_s.strip)
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
def ip_prefix(ip)
|
|
199
|
+
return nil if ip.nil? || ip.empty?
|
|
200
|
+
return ip.split(":", -1)[0, 3].join(":") if ip.include?(":")
|
|
201
|
+
|
|
202
|
+
parts = ip.split(".", -1)
|
|
203
|
+
parts.size == 4 ? parts[0, 3].join(".") : nil
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
# Handles one request. request: a Request with an absolute url. Returns a Response.
|
|
207
|
+
def handle_gate(request, deps)
|
|
208
|
+
url = URL.parse(request.url)
|
|
209
|
+
method = request.request_method.to_s.upcase
|
|
210
|
+
path = url.pathname
|
|
211
|
+
|
|
212
|
+
if path == "/.well-known/constile"
|
|
213
|
+
nonce = url.search_param("nonce")
|
|
214
|
+
confirmed = nil
|
|
215
|
+
if JS.truthy?(nonce)
|
|
216
|
+
begin
|
|
217
|
+
call_network(deps, "/verify", method: "POST", body: { "nonce" => nonce })
|
|
218
|
+
confirmed = true
|
|
219
|
+
rescue StandardError
|
|
220
|
+
confirmed = false
|
|
221
|
+
end
|
|
222
|
+
end
|
|
223
|
+
# The site's seller file, as ads.txt is for AdSense: which network sells this site, and as which seller in its /sellers.json.
|
|
224
|
+
seller_id = begin
|
|
225
|
+
get_config(deps)["seller_id"]
|
|
226
|
+
rescue StandardError
|
|
227
|
+
nil
|
|
228
|
+
end
|
|
229
|
+
network = deps.network_url.sub(%r{/\z}, "")
|
|
230
|
+
file = { "site_id" => deps.site_id, "gate_version" => GATE_VERSION, "platform" => deps.platform || "unknown",
|
|
231
|
+
"confirmed" => confirmed, "network" => network, "seller_id" => seller_id }
|
|
232
|
+
# The sellers list only when there's a seller in it: the network publishes it only when it's switched on.
|
|
233
|
+
file["sellers_json"] = "#{network}/sellers.json" unless seller_id.nil?
|
|
234
|
+
return json(file, confirmed == false ? 502 : 200)
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
if (m = CALLBACK.match(path))
|
|
238
|
+
return json({ "error" => "Method Not Allowed" }, 405) unless %w[GET POST].include?(method)
|
|
239
|
+
|
|
240
|
+
begin
|
|
241
|
+
r = call_network(deps, "/events", method: "POST", body: { "token" => m[2], "callback" => m[1], "method" => method })
|
|
242
|
+
rescue StandardError => e
|
|
243
|
+
return network_failure(e)
|
|
244
|
+
end
|
|
245
|
+
r = js_object(r)
|
|
246
|
+
return release_article(deps, r["url"], r) if JS.truthy?(r["release"])
|
|
247
|
+
|
|
248
|
+
return json(r.reject { |k, _| %w[url release release_url release_headers].include?(k) }.merge("content_released" => false))
|
|
249
|
+
end
|
|
250
|
+
|
|
251
|
+
if (m = CONTENT.match(path))
|
|
252
|
+
return json({ "error" => "Method Not Allowed" }, 405) unless %w[GET HEAD].include?(method)
|
|
253
|
+
|
|
254
|
+
begin
|
|
255
|
+
g = call_network(deps, "/grants/#{m[1]}")
|
|
256
|
+
rescue StandardError => e
|
|
257
|
+
return network_failure(e)
|
|
258
|
+
end
|
|
259
|
+
g = js_object(g)
|
|
260
|
+
unless JS.truthy?(g["release"])
|
|
261
|
+
reason = g["reason"]
|
|
262
|
+
return json({ "error" => reason.nil? ? "No access" : reason }, reason == "Commitment required" ? 403 : 410)
|
|
263
|
+
end
|
|
264
|
+
return release_article(deps, g["url"], g)
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
if (m = MACHINE_OFFER.match(path))
|
|
268
|
+
begin
|
|
269
|
+
return json(call_network(deps, "/offers/#{m[1]}"))
|
|
270
|
+
rescue StandardError => e
|
|
271
|
+
return network_failure(e)
|
|
272
|
+
end
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
# Everything below is the normal site. Only GETs can become offers.
|
|
276
|
+
return deps.origin.call(request) unless method == "GET"
|
|
277
|
+
|
|
278
|
+
begin
|
|
279
|
+
config = get_config(deps)
|
|
280
|
+
rescue StandardError => e
|
|
281
|
+
log(deps, "constile gate: no config, passing through", e)
|
|
282
|
+
return deps.origin.call(request)
|
|
283
|
+
end
|
|
284
|
+
demand_only = config["status"] == "pending" && config["count_demand"] == true
|
|
285
|
+
return deps.origin.call(request) if (config["status"] != "active" && !demand_only) || !path_eligible?(config, path)
|
|
286
|
+
|
|
287
|
+
cookies = cookie_names(request)
|
|
288
|
+
return deps.origin.call(request) if config["bypass_cookies"].any? { |c| cookies.key?(c) }
|
|
289
|
+
|
|
290
|
+
cls = Agents.classify_client(request.headers, config["agents"])
|
|
291
|
+
if cls[:kind] == "agent"
|
|
292
|
+
return deps.origin.call(request) if cls[:agent]["tier"] == "blocked"
|
|
293
|
+
|
|
294
|
+
agent_id = cls[:agent]["id"]
|
|
295
|
+
elsif cls[:kind] == "unknown_client" && config["unknown_clients"] == "offer"
|
|
296
|
+
agent_id = nil
|
|
297
|
+
else
|
|
298
|
+
return deps.origin.call(request)
|
|
299
|
+
end
|
|
300
|
+
|
|
301
|
+
if demand_only
|
|
302
|
+
# The site is in review: serve the page as usual, and tell the network an agent asked for it.
|
|
303
|
+
body = demand_request(request, url, agent_id, deps)
|
|
304
|
+
report = lambda do
|
|
305
|
+
call_network(deps, "/demand", method: "POST", body: body)
|
|
306
|
+
rescue StandardError => e
|
|
307
|
+
log(deps, "constile gate: demand report failed", e)
|
|
308
|
+
end
|
|
309
|
+
if deps.wait_until
|
|
310
|
+
deps.wait_until.call(report)
|
|
311
|
+
else
|
|
312
|
+
report.call
|
|
313
|
+
end
|
|
314
|
+
return deps.origin.call(request)
|
|
315
|
+
end
|
|
316
|
+
|
|
317
|
+
# Fetch the page first: no offer for missing pages or non-HTML, and the
|
|
318
|
+
# network needs the title and keywords to pick an ad.
|
|
319
|
+
origin_res = (deps.page_origin || deps.origin).call(request)
|
|
320
|
+
type = Headers.normalize(origin_res.header("content-type") || "")
|
|
321
|
+
return origin_res if origin_res.status.to_i != 200 || !type.match?(HTML_TYPE)
|
|
322
|
+
|
|
323
|
+
raw = origin_res.read_body
|
|
324
|
+
return origin_res if raw.nil? # a streaming body, which the adapter passes on untouched
|
|
325
|
+
|
|
326
|
+
# Keep the page's own bytes for passing it through; decode a copy, in its declared charset, to read the meta.
|
|
327
|
+
passthrough = -> { Response.new(origin_res.status, origin_res.headers, [raw]) }
|
|
328
|
+
html = Charset.decode(Body.decompress(raw, origin_res.header("content-encoding")), type)
|
|
329
|
+
meta = PageMeta.extract(html, path)
|
|
330
|
+
|
|
331
|
+
offer_req = {
|
|
332
|
+
"url" => url.to_s,
|
|
333
|
+
"title" => meta[:title],
|
|
334
|
+
"description" => meta[:description],
|
|
335
|
+
"keywords" => meta[:keywords],
|
|
336
|
+
"section" => meta[:section],
|
|
337
|
+
"agent_id" => agent_id,
|
|
338
|
+
"user_agent" => request.headers.get("user-agent") || "",
|
|
339
|
+
"signals" => signals(request, deps, agent_id),
|
|
340
|
+
}
|
|
341
|
+
# Web Bot Auth: forward the signature unchanged so the network verifies it, not the gate.
|
|
342
|
+
sig = signature(request, url)
|
|
343
|
+
offer_req["signature"] = sig if sig
|
|
344
|
+
begin
|
|
345
|
+
r = call_network(deps, "/offers", method: "POST", body: offer_req)
|
|
346
|
+
rescue StandardError => e
|
|
347
|
+
log(deps, "constile gate: offer failed, passing through", e)
|
|
348
|
+
return passthrough.call
|
|
349
|
+
end
|
|
350
|
+
r = js_object(r)
|
|
351
|
+
return passthrough.call unless JS.truthy?(r["offer"])
|
|
352
|
+
|
|
353
|
+
offer = r["offer"].is_a?(Hash) ? r["offer"] : {}
|
|
354
|
+
headers = {
|
|
355
|
+
"content-type" => offer["content_type"].to_s,
|
|
356
|
+
"constile-offer" => offer["offer_url"].to_s,
|
|
357
|
+
"link" => "<#{offer["offer_url"]}>; rel=\"sponsored-access\"",
|
|
358
|
+
}.merge(SPONSORED_HEADERS)
|
|
359
|
+
Response.new(200, headers, [offer["body"].nil? ? "" : offer["body"].to_s])
|
|
360
|
+
end
|
|
361
|
+
|
|
362
|
+
def json(obj, status = 200)
|
|
363
|
+
Response.new(status, { "content-type" => "application/json" }.merge(SPONSORED_HEADERS), [JSON.generate(obj)])
|
|
364
|
+
end
|
|
365
|
+
|
|
366
|
+
def network_failure(error)
|
|
367
|
+
return json({ "error" => error.message }, error.status) if error.is_a?(NetworkError)
|
|
368
|
+
|
|
369
|
+
json({ "error" => UNAVAILABLE }, 503)
|
|
370
|
+
end
|
|
371
|
+
|
|
372
|
+
# Property access as JavaScript does it on a parsed JSON value: null throws, other scalars have no properties.
|
|
373
|
+
def js_object(value)
|
|
374
|
+
case value
|
|
375
|
+
when Hash then value
|
|
376
|
+
when nil then raise TypeError, "Cannot read properties of null"
|
|
377
|
+
when Array then value.each_with_index.map { |v, i| [i.to_s, v] }.to_h
|
|
378
|
+
else {}
|
|
379
|
+
end
|
|
380
|
+
end
|
|
381
|
+
|
|
382
|
+
def call_network(deps, path, method: "GET", body: nil)
|
|
383
|
+
headers = {
|
|
384
|
+
"Constile-Site" => deps.site_id.to_s,
|
|
385
|
+
"Authorization" => "Bearer #{deps.secret}",
|
|
386
|
+
"Constile-Gate-Version" => GATE_VERSION,
|
|
387
|
+
"Constile-Gate-Platform" => (deps.platform || "unknown").to_s,
|
|
388
|
+
}
|
|
389
|
+
headers["Content-Type"] = "application/json" unless body.nil?
|
|
390
|
+
req = NetworkRequest.new(method, "#{deps.network_url.sub(%r{/\z}, "")}/v1/gate#{path}", headers, body.nil? ? nil : JSON.generate(body))
|
|
391
|
+
res = (deps.network || HTTPClient.new).call(req)
|
|
392
|
+
text = res.body.to_s.dup.force_encoding(Encoding::UTF_8)
|
|
393
|
+
unless (200..299).cover?(res.status.to_i)
|
|
394
|
+
detail = begin
|
|
395
|
+
JSON.parse(text)
|
|
396
|
+
rescue StandardError
|
|
397
|
+
{}
|
|
398
|
+
end
|
|
399
|
+
raise TypeError, "Cannot read properties of null" if detail.nil?
|
|
400
|
+
|
|
401
|
+
error = detail.is_a?(Hash) ? detail["error"] : nil
|
|
402
|
+
raise NetworkError.new(res.status.to_i, error.nil? ? "network returned #{res.status}" : JS.to_s(error))
|
|
403
|
+
end
|
|
404
|
+
JSON.parse(text)
|
|
405
|
+
end
|
|
406
|
+
|
|
407
|
+
# The site's config: from the cache while it's fresh; past its TTL, the last config while a
|
|
408
|
+
# new one is fetched in the background (for up to a day), so people never wait on the network
|
|
409
|
+
# once the gate has a config. Requests with no config share one fetch, and after a failure the
|
|
410
|
+
# gate doesn't ask again for config_retry_s.
|
|
411
|
+
def get_config(deps)
|
|
412
|
+
key = "#{deps.network_url}|#{deps.site_id}"
|
|
413
|
+
now = clock(deps)
|
|
414
|
+
hit = ConfigCache.get(key)
|
|
415
|
+
return hit[:config] if hit && hit[:until] > now
|
|
416
|
+
|
|
417
|
+
stale = hit && hit[:until] + CONFIG_STALE_S > now ? hit[:config] : nil
|
|
418
|
+
if (ConfigCache.failed_until(key) || 0) > now
|
|
419
|
+
return stale unless stale.nil?
|
|
420
|
+
|
|
421
|
+
raise "the network failed recently; passing through until it's retried"
|
|
422
|
+
end
|
|
423
|
+
unless stale.nil?
|
|
424
|
+
refresh_config(deps, key) if ConfigCache.start_refresh(key)
|
|
425
|
+
return stale
|
|
426
|
+
end
|
|
427
|
+
fetch, owner = ConfigCache.claim_fetch(key)
|
|
428
|
+
return ConfigCache.wait_fetch(fetch) unless owner
|
|
429
|
+
|
|
430
|
+
begin
|
|
431
|
+
config = fetch_config(deps, key)
|
|
432
|
+
rescue Exception => e # rubocop:disable Lint/RescueException -- waiters must never hang, whatever stopped the fetch
|
|
433
|
+
ConfigCache.finish_fetch(key, fetch, nil, e.is_a?(StandardError) ? e : RuntimeError.new("the config fetch stopped"))
|
|
434
|
+
raise
|
|
435
|
+
end
|
|
436
|
+
ConfigCache.finish_fetch(key, fetch, config, nil)
|
|
437
|
+
config
|
|
438
|
+
end
|
|
439
|
+
|
|
440
|
+
def clock(deps)
|
|
441
|
+
deps.now ? deps.now.call : Time.now.to_f
|
|
442
|
+
end
|
|
443
|
+
|
|
444
|
+
def fetch_config(deps, key)
|
|
445
|
+
begin
|
|
446
|
+
config = call_network(deps, "/config")
|
|
447
|
+
rescue StandardError
|
|
448
|
+
config_failed(deps, key)
|
|
449
|
+
raise
|
|
450
|
+
end
|
|
451
|
+
ConfigCache.recover(key)
|
|
452
|
+
ttl = config.key?("ttl_s") ? JS.number(config["ttl_s"]) : Float::NAN
|
|
453
|
+
# A ttl_s that isn't a number caches nothing, as Math.max(10, NaN) does in the reference.
|
|
454
|
+
ConfigCache.set(key, config, clock(deps) + ttl.clamp(10, 3600)) unless ttl.is_a?(Float) && ttl.nan?
|
|
455
|
+
config
|
|
456
|
+
end
|
|
457
|
+
|
|
458
|
+
# The wait starts when the fetch fails, not when it began.
|
|
459
|
+
def config_failed(deps, key)
|
|
460
|
+
ConfigCache.fail(key, clock(deps) + (deps.config_retry_s.nil? ? CONFIG_RETRY_S : deps.config_retry_s))
|
|
461
|
+
end
|
|
462
|
+
|
|
463
|
+
# Fetches a fresh config after the response (with wait_until), or now without it. A refresh the
|
|
464
|
+
# background queue can't take counts as a failure, so the next is tried after config_retry_s.
|
|
465
|
+
def refresh_config(deps, key)
|
|
466
|
+
job = lambda do
|
|
467
|
+
fetch_config(deps, key)
|
|
468
|
+
rescue StandardError => e
|
|
469
|
+
log(deps, "constile gate: config refresh failed", e)
|
|
470
|
+
ensure
|
|
471
|
+
ConfigCache.end_refresh(key)
|
|
472
|
+
end
|
|
473
|
+
if deps.wait_until
|
|
474
|
+
return unless deps.wait_until.call(job) == false
|
|
475
|
+
|
|
476
|
+
config_failed(deps, key)
|
|
477
|
+
ConfigCache.end_refresh(key)
|
|
478
|
+
else
|
|
479
|
+
job.call
|
|
480
|
+
end
|
|
481
|
+
end
|
|
482
|
+
|
|
483
|
+
def cookie_names(request)
|
|
484
|
+
names = {}
|
|
485
|
+
(request.headers.get("cookie") || "").split(";", -1).each do |part|
|
|
486
|
+
name = JS.trim(part.split("=", -1)[0] || "")
|
|
487
|
+
names[name] = true unless name.empty?
|
|
488
|
+
end
|
|
489
|
+
names
|
|
490
|
+
end
|
|
491
|
+
|
|
492
|
+
def path_eligible?(config, path)
|
|
493
|
+
config["eligible_paths"].any? do |p|
|
|
494
|
+
p == "/" || path == p.sub(%r{/\z}, "") || path.start_with?(p.end_with?("/") ? p : "#{p}/")
|
|
495
|
+
end
|
|
496
|
+
end
|
|
497
|
+
|
|
498
|
+
def signal(deps, name)
|
|
499
|
+
s = deps.signals
|
|
500
|
+
return nil unless s
|
|
501
|
+
|
|
502
|
+
s[name].nil? ? s[name.to_s] : s[name]
|
|
503
|
+
end
|
|
504
|
+
|
|
505
|
+
def signals(request, deps, agent_id)
|
|
506
|
+
{
|
|
507
|
+
"asn" => signal(deps, :asn),
|
|
508
|
+
"as_org" => signal(deps, :as_org),
|
|
509
|
+
"country" => signal(deps, :country),
|
|
510
|
+
"ip_prefix" => signal(deps, :ip_prefix),
|
|
511
|
+
"signature_agent" => request.headers.get("signature-agent"),
|
|
512
|
+
# Only a registered agent's address goes to the network, to check against the operator's list.
|
|
513
|
+
"ip" => JS.truthy?(agent_id) ? signal(deps, :ip) : nil,
|
|
514
|
+
}
|
|
515
|
+
end
|
|
516
|
+
|
|
517
|
+
def signature(request, url)
|
|
518
|
+
sig = request.headers.get("signature")
|
|
519
|
+
sig_input = request.headers.get("signature-input")
|
|
520
|
+
return nil unless JS.truthy?(sig) && JS.truthy?(sig_input)
|
|
521
|
+
|
|
522
|
+
{ "signature" => sig, "signature_input" => sig_input, "signature_agent" => request.headers.get("signature-agent"), "authority" => url.host }
|
|
523
|
+
end
|
|
524
|
+
|
|
525
|
+
def demand_request(request, url, agent_id, deps)
|
|
526
|
+
out = { "url" => url.to_s, "agent_id" => agent_id, "user_agent" => request.headers.get("user-agent") || "",
|
|
527
|
+
"signals" => signals(request, deps, agent_id) }
|
|
528
|
+
sig = signature(request, url)
|
|
529
|
+
out["signature"] = sig if sig
|
|
530
|
+
out
|
|
531
|
+
end
|
|
532
|
+
|
|
533
|
+
# X- names the network can't add. Servers fold `_` and `-` into the same variable (WSGI, Rack, PHP), so names with `_`
|
|
534
|
+
# are refused outright. The rest steer routing, the method, middleware, or who the origin thinks the caller is.
|
|
535
|
+
DENIED_X_PREFIXES = %w[x-forwarded x-original x-rewrite x-middleware x-real x-client x-cluster-client x-originating x-remote x-host x-http-method x-method].freeze
|
|
536
|
+
DENIED_X_SUFFIXES = %w[-override -ip -user -host -url -uri -path -proto -scheme -port -for].freeze
|
|
537
|
+
HEADER_TOKEN = /\A[!#$%&'*+\-.^_`|~0-9A-Za-z]+\z/.freeze
|
|
538
|
+
|
|
539
|
+
# Header names the network may add to a release: its paywall key, credentials for the page, and custom X- headers. Nothing else.
|
|
540
|
+
def allowed_release_header?(name)
|
|
541
|
+
return false if name.include?("_")
|
|
542
|
+
|
|
543
|
+
n = name.downcase
|
|
544
|
+
return true if %w[cookie authorization constile-paywall-key].include?(n)
|
|
545
|
+
return false unless n.start_with?("x-")
|
|
546
|
+
|
|
547
|
+
DENIED_X_PREFIXES.none? { |p| n.start_with?(p) } && DENIED_X_SUFFIXES.none? { |x| n.end_with?(x) }
|
|
548
|
+
end
|
|
549
|
+
|
|
550
|
+
# Whether a header can carry this name and value, as the Fetch API's Headers decides.
|
|
551
|
+
def header_carries?(name, value)
|
|
552
|
+
return false unless name.valid_encoding? && value.valid_encoding?
|
|
553
|
+
return false unless HEADER_TOKEN.match?(name)
|
|
554
|
+
return false if value.match?(/[\0\r\n]/)
|
|
555
|
+
|
|
556
|
+
value.encode(Encoding::UTF_8).each_char.all? { |c| c.ord <= 0xFF }
|
|
557
|
+
rescue ArgumentError, EncodingError
|
|
558
|
+
false
|
|
559
|
+
end
|
|
560
|
+
|
|
561
|
+
# The URL to fetch: release_url only on the page's own scheme, host, and port, with no username or password.
|
|
562
|
+
def release_target(url, release_url)
|
|
563
|
+
page = URL.parse(url)
|
|
564
|
+
return page.to_s unless release_url.is_a?(String)
|
|
565
|
+
|
|
566
|
+
wanted = URL.parse(release_url)
|
|
567
|
+
# `host` includes a non-default port, so another port is another host. Userinfo is never accepted.
|
|
568
|
+
return wanted.to_s if wanted.scheme == page.scheme && wanted.host == page.host && !wanted.credentials?
|
|
569
|
+
|
|
570
|
+
page.to_s
|
|
571
|
+
rescue ArgumentError
|
|
572
|
+
URL.parse(url).to_s
|
|
573
|
+
end
|
|
574
|
+
|
|
575
|
+
# Fetches the article for an agent that has access. Never forwards the agent's headers to the origin.
|
|
576
|
+
# For a paywalled page the network's answer adds release instructions: a `release_url` (used only on the page's own
|
|
577
|
+
# scheme, host, and port) and `release_headers` (only the allowed names; the rest are dropped without a trace, and no
|
|
578
|
+
# name or value is logged).
|
|
579
|
+
def release_article(deps, url, instructions = nil)
|
|
580
|
+
instructions = {} unless instructions.is_a?(Hash)
|
|
581
|
+
sent = { "accept" => RELEASE_ACCEPT, "user-agent" => "#{GATE_VERSION} (release)" }
|
|
582
|
+
extra = instructions["release_headers"]
|
|
583
|
+
if extra.is_a?(Hash)
|
|
584
|
+
extra.each do |name, value|
|
|
585
|
+
next unless name.is_a?(String) && value.is_a?(String) && header_carries?(name, value) && allowed_release_header?(name)
|
|
586
|
+
|
|
587
|
+
sent[name.downcase] = value
|
|
588
|
+
end
|
|
589
|
+
end
|
|
590
|
+
res = deps.origin.call(Request.new(
|
|
591
|
+
request_method: "GET",
|
|
592
|
+
url: release_target(url, instructions["release_url"]),
|
|
593
|
+
headers: Headers.new(sent)
|
|
594
|
+
))
|
|
595
|
+
replaced = SPONSORED_HEADERS.keys + ["set-cookie"]
|
|
596
|
+
headers = {}
|
|
597
|
+
res.headers.each { |k, v| headers[k] = v unless replaced.include?(k.to_s.downcase) }
|
|
598
|
+
Response.new(res.status, headers.merge(SPONSORED_HEADERS), res.body)
|
|
599
|
+
end
|
|
600
|
+
|
|
601
|
+
def log(deps, message, error = nil)
|
|
602
|
+
text = error ? "#{message}: #{error.class}: #{error.message}" : message
|
|
603
|
+
logger = deps.logger
|
|
604
|
+
if logger.respond_to?(:warn)
|
|
605
|
+
logger.warn(text)
|
|
606
|
+
elsif logger.respond_to?(:call)
|
|
607
|
+
logger.call(text)
|
|
608
|
+
else
|
|
609
|
+
Kernel.warn(text)
|
|
610
|
+
end
|
|
611
|
+
end
|
|
612
|
+
end
|