constile_gate 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,612 @@
1
+ require "json"
2
+ require "constile_gate/version"
3
+ require "constile_gate/js"
4
+ require "constile_gate/url"
5
+ require "constile_gate/page_meta"
6
+ require "constile_gate/agents"
7
+ require "constile_gate/charset"
8
+ require "constile_gate/http"
9
+
10
+ # The publisher's gate, ported from packages/gate/src/core.ts. It sits in front of
11
+ # the publisher's site and does three things:
12
+ #
13
+ # 1. For an agent fetching an eligible page for a person, it serves the
14
+ # Constile offer page with HTTP 200 instead of the article.
15
+ # 2. It forwards the offer's callbacks (/sponsored/...) to the network and, when
16
+ # the network grants access, returns the article from the origin.
17
+ # 3. Everything else (people, search engines, crawlers, subscribers, other paths)
18
+ # goes straight to the origin, untouched.
19
+ #
20
+ # It fails open: if the network can't be reached, the page is served as if the
21
+ # gate weren't there. Callbacks can't fail open, because they release content
22
+ # only against a recorded commitment.
23
+ module ConstileGate
24
+ # What the gate needs for one request.
25
+ # site_id, secret, network_url the site's settings; network_url is the network's origin
26
+ # origin callable(Request) -> Response: fetches from the publisher's site
27
+ # page_origin callable(Request) -> Response: fetches the page an offer is for; defaults to origin.
28
+ # An in-process adapter uses it to ask the app for the page uncompressed.
29
+ # network callable(NetworkRequest) -> NetworkResponse; defaults to HTTPClient
30
+ # signals { asn:, as_org:, country:, ip_prefix:, ip: }; ip is sent only for registered agents
31
+ # now callable -> seconds since the epoch
32
+ # config_retry_s after a failed config fetch, seconds to pass everything through before asking again (default 30)
33
+ # wait_until callable(job): runs job (a callable) after the response; without it, the job runs first
34
+ # platform where the gate runs, such as "rack" or "rails"
35
+ # logger callable(message) or an object with #warn; defaults to Kernel#warn
36
+ Deps = Struct.new(:site_id, :secret, :network_url, :origin, :page_origin, :network, :signals, :now, :config_retry_s,
37
+ :wait_until, :platform, :logger, keyword_init: true) do
38
+ # The secret never prints: not in logs, error pages, or pp.
39
+ def inspect
40
+ fields = members.map { |m| "#{m}=#{m == :secret && !self[m].nil? ? "[REDACTED]" : self[m].inspect}" }
41
+ "#<struct #{self.class.name} #{fields.join(", ")}>"
42
+ end
43
+ alias_method :to_s, :inspect
44
+
45
+ def pretty_print(q)
46
+ q.text(inspect)
47
+ end
48
+ end
49
+
50
+ # The network answered with an error status. The message is the network's error.
51
+ class NetworkError < StandardError
52
+ attr_reader :status
53
+
54
+ def initialize(status, message)
55
+ super(message)
56
+ @status = status
57
+ end
58
+ end
59
+
60
+ SPONSORED_HEADERS = { "cache-control" => "no-store", "x-robots-tag" => "noindex, nofollow", "vary" => "User-Agent" }.freeze
61
+ CALLBACK = %r{\A/sponsored/(deferred|prepared|delivered)/([A-Za-z0-9_-]+)\z}.freeze
62
+ CONTENT = %r{\A/sponsored/content/([A-Za-z0-9_-]+)\z}.freeze
63
+ MACHINE_OFFER = %r{\A/sponsored/offer/([A-Za-z0-9_-]+)\z}.freeze
64
+ HTML_TYPE = /#{PageMeta.ci("text/html")}|#{PageMeta.ci("application/xhtml")}/.freeze
65
+ RELEASE_ACCEPT = "text/html,application/xhtml+xml,text/plain;q=0.9,*/*;q=0.5".freeze
66
+ UNAVAILABLE = "Sponsored access is unavailable right now".freeze
67
+
68
+ CONFIG_RETRY_S = 30
69
+ # How long a config may be used past its TTL while the network can't give a fresh one.
70
+ CONFIG_STALE_S = 86_400
71
+ # How long a request waits for a fetch another request started, at most. A fetch or refresh
72
+ # older than this (or one a parent process started before forking) is treated as gone.
73
+ CONFIG_WAIT_S = 30
74
+
75
+ # Configs from the network, per network and site, kept for their ttl_s (10 to 3,600 seconds),
76
+ # and when the network last failed to give one: the gate leaves it alone until then, so
77
+ # people aren't held up. Also the fetches in flight: requests with no config share one, and
78
+ # one background refresh at a time keeps a stale config fresh.
79
+ module ConfigCache
80
+ # A fetch for requests with no config: the first request fetches, the others wait for its result.
81
+ Fetch = Struct.new(:started, :pid, :done, :config, :error, :cond)
82
+
83
+ @entries = {}
84
+ @failures = {}
85
+ @fetches = {}
86
+ @refreshing = {}
87
+ @mutex = Mutex.new
88
+
89
+ def self.clock
90
+ Process.clock_gettime(Process::CLOCK_MONOTONIC)
91
+ end
92
+
93
+ def self.live?(started, pid)
94
+ pid == Process.pid && clock - started <= CONFIG_WAIT_S
95
+ end
96
+
97
+ def self.get(key)
98
+ @mutex.synchronize { @entries[key] }
99
+ end
100
+
101
+ def self.set(key, config, until_s)
102
+ @mutex.synchronize { @entries[key] = { config: config, until: until_s } }
103
+ end
104
+
105
+ def self.failed_until(key)
106
+ @mutex.synchronize { @failures[key] }
107
+ end
108
+
109
+ def self.fail(key, until_s)
110
+ @mutex.synchronize { @failures[key] = until_s }
111
+ end
112
+
113
+ def self.recover(key)
114
+ @mutex.synchronize { @failures.delete(key) }
115
+ end
116
+
117
+ # [fetch, owner]: the fetch in flight for key, or a new one this request owns.
118
+ def self.claim_fetch(key)
119
+ @mutex.synchronize do
120
+ fetch = @fetches[key]
121
+ return [fetch, false] if fetch && live?(fetch.started, fetch.pid)
122
+
123
+ fetch = Fetch.new(clock, Process.pid, false, nil, nil, ConditionVariable.new)
124
+ @fetches[key] = fetch
125
+ [fetch, true]
126
+ end
127
+ end
128
+
129
+ def self.finish_fetch(key, fetch, config, error)
130
+ @mutex.synchronize do
131
+ fetch.config = config
132
+ fetch.error = error
133
+ fetch.done = true
134
+ @fetches.delete(key) if @fetches[key].equal?(fetch)
135
+ fetch.cond.broadcast
136
+ end
137
+ end
138
+
139
+ # The shared fetch's config, or its error raised here.
140
+ def self.wait_fetch(fetch)
141
+ deadline = clock + CONFIG_WAIT_S
142
+ @mutex.synchronize do
143
+ until fetch.done
144
+ left = deadline - clock
145
+ raise "the shared config fetch didn't finish within #{CONFIG_WAIT_S} s" if left <= 0
146
+
147
+ fetch.cond.wait(@mutex, left)
148
+ end
149
+ end
150
+ raise fetch.error if fetch.error
151
+
152
+ fetch.config
153
+ end
154
+
155
+ # True when this request should start the refresh: none is running for key.
156
+ def self.start_refresh(key)
157
+ @mutex.synchronize do
158
+ started = @refreshing[key]
159
+ return false if started && live?(*started)
160
+
161
+ @refreshing[key] = [clock, Process.pid]
162
+ true
163
+ end
164
+ end
165
+
166
+ def self.end_refresh(key)
167
+ @mutex.synchronize { @refreshing.delete(key) }
168
+ end
169
+
170
+ def self.clear
171
+ @mutex.synchronize do
172
+ @entries.clear
173
+ @failures.clear
174
+ @fetches.clear
175
+ @refreshing.clear
176
+ end
177
+ end
178
+ end
179
+
180
+ module_function
181
+
182
+ # For tests: forget cached configs and failures.
183
+ def clear_gate_cache
184
+ ConfigCache.clear
185
+ end
186
+
187
+ # Whether the gate itself answers this path: the seller file and the /sponsored/ routes.
188
+ def gate_route?(path)
189
+ path == "/.well-known/constile" || CALLBACK.match?(path) || CONTENT.match?(path) || MACHINE_OFFER.match?(path)
190
+ end
191
+
192
+ # The first three parts of an address, as in the research gateway's request log.
193
+ # CONSTILE_DISABLED's on values: 1, true, yes, or on, in any case.
194
+ def on?(value)
195
+ /\A(1|true|yes|on)\z/i.match?(value.to_s.strip)
196
+ end
197
+
198
+ def ip_prefix(ip)
199
+ return nil if ip.nil? || ip.empty?
200
+ return ip.split(":", -1)[0, 3].join(":") if ip.include?(":")
201
+
202
+ parts = ip.split(".", -1)
203
+ parts.size == 4 ? parts[0, 3].join(".") : nil
204
+ end
205
+
206
+ # Handles one request. request: a Request with an absolute url. Returns a Response.
207
+ def handle_gate(request, deps)
208
+ url = URL.parse(request.url)
209
+ method = request.request_method.to_s.upcase
210
+ path = url.pathname
211
+
212
+ if path == "/.well-known/constile"
213
+ nonce = url.search_param("nonce")
214
+ confirmed = nil
215
+ if JS.truthy?(nonce)
216
+ begin
217
+ call_network(deps, "/verify", method: "POST", body: { "nonce" => nonce })
218
+ confirmed = true
219
+ rescue StandardError
220
+ confirmed = false
221
+ end
222
+ end
223
+ # The site's seller file, as ads.txt is for AdSense: which network sells this site, and as which seller in its /sellers.json.
224
+ seller_id = begin
225
+ get_config(deps)["seller_id"]
226
+ rescue StandardError
227
+ nil
228
+ end
229
+ network = deps.network_url.sub(%r{/\z}, "")
230
+ file = { "site_id" => deps.site_id, "gate_version" => GATE_VERSION, "platform" => deps.platform || "unknown",
231
+ "confirmed" => confirmed, "network" => network, "seller_id" => seller_id }
232
+ # The sellers list only when there's a seller in it: the network publishes it only when it's switched on.
233
+ file["sellers_json"] = "#{network}/sellers.json" unless seller_id.nil?
234
+ return json(file, confirmed == false ? 502 : 200)
235
+ end
236
+
237
+ if (m = CALLBACK.match(path))
238
+ return json({ "error" => "Method Not Allowed" }, 405) unless %w[GET POST].include?(method)
239
+
240
+ begin
241
+ r = call_network(deps, "/events", method: "POST", body: { "token" => m[2], "callback" => m[1], "method" => method })
242
+ rescue StandardError => e
243
+ return network_failure(e)
244
+ end
245
+ r = js_object(r)
246
+ return release_article(deps, r["url"], r) if JS.truthy?(r["release"])
247
+
248
+ return json(r.reject { |k, _| %w[url release release_url release_headers].include?(k) }.merge("content_released" => false))
249
+ end
250
+
251
+ if (m = CONTENT.match(path))
252
+ return json({ "error" => "Method Not Allowed" }, 405) unless %w[GET HEAD].include?(method)
253
+
254
+ begin
255
+ g = call_network(deps, "/grants/#{m[1]}")
256
+ rescue StandardError => e
257
+ return network_failure(e)
258
+ end
259
+ g = js_object(g)
260
+ unless JS.truthy?(g["release"])
261
+ reason = g["reason"]
262
+ return json({ "error" => reason.nil? ? "No access" : reason }, reason == "Commitment required" ? 403 : 410)
263
+ end
264
+ return release_article(deps, g["url"], g)
265
+ end
266
+
267
+ if (m = MACHINE_OFFER.match(path))
268
+ begin
269
+ return json(call_network(deps, "/offers/#{m[1]}"))
270
+ rescue StandardError => e
271
+ return network_failure(e)
272
+ end
273
+ end
274
+
275
+ # Everything below is the normal site. Only GETs can become offers.
276
+ return deps.origin.call(request) unless method == "GET"
277
+
278
+ begin
279
+ config = get_config(deps)
280
+ rescue StandardError => e
281
+ log(deps, "constile gate: no config, passing through", e)
282
+ return deps.origin.call(request)
283
+ end
284
+ demand_only = config["status"] == "pending" && config["count_demand"] == true
285
+ return deps.origin.call(request) if (config["status"] != "active" && !demand_only) || !path_eligible?(config, path)
286
+
287
+ cookies = cookie_names(request)
288
+ return deps.origin.call(request) if config["bypass_cookies"].any? { |c| cookies.key?(c) }
289
+
290
+ cls = Agents.classify_client(request.headers, config["agents"])
291
+ if cls[:kind] == "agent"
292
+ return deps.origin.call(request) if cls[:agent]["tier"] == "blocked"
293
+
294
+ agent_id = cls[:agent]["id"]
295
+ elsif cls[:kind] == "unknown_client" && config["unknown_clients"] == "offer"
296
+ agent_id = nil
297
+ else
298
+ return deps.origin.call(request)
299
+ end
300
+
301
+ if demand_only
302
+ # The site is in review: serve the page as usual, and tell the network an agent asked for it.
303
+ body = demand_request(request, url, agent_id, deps)
304
+ report = lambda do
305
+ call_network(deps, "/demand", method: "POST", body: body)
306
+ rescue StandardError => e
307
+ log(deps, "constile gate: demand report failed", e)
308
+ end
309
+ if deps.wait_until
310
+ deps.wait_until.call(report)
311
+ else
312
+ report.call
313
+ end
314
+ return deps.origin.call(request)
315
+ end
316
+
317
+ # Fetch the page first: no offer for missing pages or non-HTML, and the
318
+ # network needs the title and keywords to pick an ad.
319
+ origin_res = (deps.page_origin || deps.origin).call(request)
320
+ type = Headers.normalize(origin_res.header("content-type") || "")
321
+ return origin_res if origin_res.status.to_i != 200 || !type.match?(HTML_TYPE)
322
+
323
+ raw = origin_res.read_body
324
+ return origin_res if raw.nil? # a streaming body, which the adapter passes on untouched
325
+
326
+ # Keep the page's own bytes for passing it through; decode a copy, in its declared charset, to read the meta.
327
+ passthrough = -> { Response.new(origin_res.status, origin_res.headers, [raw]) }
328
+ html = Charset.decode(Body.decompress(raw, origin_res.header("content-encoding")), type)
329
+ meta = PageMeta.extract(html, path)
330
+
331
+ offer_req = {
332
+ "url" => url.to_s,
333
+ "title" => meta[:title],
334
+ "description" => meta[:description],
335
+ "keywords" => meta[:keywords],
336
+ "section" => meta[:section],
337
+ "agent_id" => agent_id,
338
+ "user_agent" => request.headers.get("user-agent") || "",
339
+ "signals" => signals(request, deps, agent_id),
340
+ }
341
+ # Web Bot Auth: forward the signature unchanged so the network verifies it, not the gate.
342
+ sig = signature(request, url)
343
+ offer_req["signature"] = sig if sig
344
+ begin
345
+ r = call_network(deps, "/offers", method: "POST", body: offer_req)
346
+ rescue StandardError => e
347
+ log(deps, "constile gate: offer failed, passing through", e)
348
+ return passthrough.call
349
+ end
350
+ r = js_object(r)
351
+ return passthrough.call unless JS.truthy?(r["offer"])
352
+
353
+ offer = r["offer"].is_a?(Hash) ? r["offer"] : {}
354
+ headers = {
355
+ "content-type" => offer["content_type"].to_s,
356
+ "constile-offer" => offer["offer_url"].to_s,
357
+ "link" => "<#{offer["offer_url"]}>; rel=\"sponsored-access\"",
358
+ }.merge(SPONSORED_HEADERS)
359
+ Response.new(200, headers, [offer["body"].nil? ? "" : offer["body"].to_s])
360
+ end
361
+
362
+ def json(obj, status = 200)
363
+ Response.new(status, { "content-type" => "application/json" }.merge(SPONSORED_HEADERS), [JSON.generate(obj)])
364
+ end
365
+
366
+ def network_failure(error)
367
+ return json({ "error" => error.message }, error.status) if error.is_a?(NetworkError)
368
+
369
+ json({ "error" => UNAVAILABLE }, 503)
370
+ end
371
+
372
+ # Property access as JavaScript does it on a parsed JSON value: null throws, other scalars have no properties.
373
+ def js_object(value)
374
+ case value
375
+ when Hash then value
376
+ when nil then raise TypeError, "Cannot read properties of null"
377
+ when Array then value.each_with_index.map { |v, i| [i.to_s, v] }.to_h
378
+ else {}
379
+ end
380
+ end
381
+
382
+ def call_network(deps, path, method: "GET", body: nil)
383
+ headers = {
384
+ "Constile-Site" => deps.site_id.to_s,
385
+ "Authorization" => "Bearer #{deps.secret}",
386
+ "Constile-Gate-Version" => GATE_VERSION,
387
+ "Constile-Gate-Platform" => (deps.platform || "unknown").to_s,
388
+ }
389
+ headers["Content-Type"] = "application/json" unless body.nil?
390
+ req = NetworkRequest.new(method, "#{deps.network_url.sub(%r{/\z}, "")}/v1/gate#{path}", headers, body.nil? ? nil : JSON.generate(body))
391
+ res = (deps.network || HTTPClient.new).call(req)
392
+ text = res.body.to_s.dup.force_encoding(Encoding::UTF_8)
393
+ unless (200..299).cover?(res.status.to_i)
394
+ detail = begin
395
+ JSON.parse(text)
396
+ rescue StandardError
397
+ {}
398
+ end
399
+ raise TypeError, "Cannot read properties of null" if detail.nil?
400
+
401
+ error = detail.is_a?(Hash) ? detail["error"] : nil
402
+ raise NetworkError.new(res.status.to_i, error.nil? ? "network returned #{res.status}" : JS.to_s(error))
403
+ end
404
+ JSON.parse(text)
405
+ end
406
+
407
+ # The site's config: from the cache while it's fresh; past its TTL, the last config while a
408
+ # new one is fetched in the background (for up to a day), so people never wait on the network
409
+ # once the gate has a config. Requests with no config share one fetch, and after a failure the
410
+ # gate doesn't ask again for config_retry_s.
411
+ def get_config(deps)
412
+ key = "#{deps.network_url}|#{deps.site_id}"
413
+ now = clock(deps)
414
+ hit = ConfigCache.get(key)
415
+ return hit[:config] if hit && hit[:until] > now
416
+
417
+ stale = hit && hit[:until] + CONFIG_STALE_S > now ? hit[:config] : nil
418
+ if (ConfigCache.failed_until(key) || 0) > now
419
+ return stale unless stale.nil?
420
+
421
+ raise "the network failed recently; passing through until it's retried"
422
+ end
423
+ unless stale.nil?
424
+ refresh_config(deps, key) if ConfigCache.start_refresh(key)
425
+ return stale
426
+ end
427
+ fetch, owner = ConfigCache.claim_fetch(key)
428
+ return ConfigCache.wait_fetch(fetch) unless owner
429
+
430
+ begin
431
+ config = fetch_config(deps, key)
432
+ rescue Exception => e # rubocop:disable Lint/RescueException -- waiters must never hang, whatever stopped the fetch
433
+ ConfigCache.finish_fetch(key, fetch, nil, e.is_a?(StandardError) ? e : RuntimeError.new("the config fetch stopped"))
434
+ raise
435
+ end
436
+ ConfigCache.finish_fetch(key, fetch, config, nil)
437
+ config
438
+ end
439
+
440
+ def clock(deps)
441
+ deps.now ? deps.now.call : Time.now.to_f
442
+ end
443
+
444
+ def fetch_config(deps, key)
445
+ begin
446
+ config = call_network(deps, "/config")
447
+ rescue StandardError
448
+ config_failed(deps, key)
449
+ raise
450
+ end
451
+ ConfigCache.recover(key)
452
+ ttl = config.key?("ttl_s") ? JS.number(config["ttl_s"]) : Float::NAN
453
+ # A ttl_s that isn't a number caches nothing, as Math.max(10, NaN) does in the reference.
454
+ ConfigCache.set(key, config, clock(deps) + ttl.clamp(10, 3600)) unless ttl.is_a?(Float) && ttl.nan?
455
+ config
456
+ end
457
+
458
+ # The wait starts when the fetch fails, not when it began.
459
+ def config_failed(deps, key)
460
+ ConfigCache.fail(key, clock(deps) + (deps.config_retry_s.nil? ? CONFIG_RETRY_S : deps.config_retry_s))
461
+ end
462
+
463
+ # Fetches a fresh config after the response (with wait_until), or now without it. A refresh the
464
+ # background queue can't take counts as a failure, so the next is tried after config_retry_s.
465
+ def refresh_config(deps, key)
466
+ job = lambda do
467
+ fetch_config(deps, key)
468
+ rescue StandardError => e
469
+ log(deps, "constile gate: config refresh failed", e)
470
+ ensure
471
+ ConfigCache.end_refresh(key)
472
+ end
473
+ if deps.wait_until
474
+ return unless deps.wait_until.call(job) == false
475
+
476
+ config_failed(deps, key)
477
+ ConfigCache.end_refresh(key)
478
+ else
479
+ job.call
480
+ end
481
+ end
482
+
483
+ def cookie_names(request)
484
+ names = {}
485
+ (request.headers.get("cookie") || "").split(";", -1).each do |part|
486
+ name = JS.trim(part.split("=", -1)[0] || "")
487
+ names[name] = true unless name.empty?
488
+ end
489
+ names
490
+ end
491
+
492
+ def path_eligible?(config, path)
493
+ config["eligible_paths"].any? do |p|
494
+ p == "/" || path == p.sub(%r{/\z}, "") || path.start_with?(p.end_with?("/") ? p : "#{p}/")
495
+ end
496
+ end
497
+
498
+ def signal(deps, name)
499
+ s = deps.signals
500
+ return nil unless s
501
+
502
+ s[name].nil? ? s[name.to_s] : s[name]
503
+ end
504
+
505
+ def signals(request, deps, agent_id)
506
+ {
507
+ "asn" => signal(deps, :asn),
508
+ "as_org" => signal(deps, :as_org),
509
+ "country" => signal(deps, :country),
510
+ "ip_prefix" => signal(deps, :ip_prefix),
511
+ "signature_agent" => request.headers.get("signature-agent"),
512
+ # Only a registered agent's address goes to the network, to check against the operator's list.
513
+ "ip" => JS.truthy?(agent_id) ? signal(deps, :ip) : nil,
514
+ }
515
+ end
516
+
517
+ def signature(request, url)
518
+ sig = request.headers.get("signature")
519
+ sig_input = request.headers.get("signature-input")
520
+ return nil unless JS.truthy?(sig) && JS.truthy?(sig_input)
521
+
522
+ { "signature" => sig, "signature_input" => sig_input, "signature_agent" => request.headers.get("signature-agent"), "authority" => url.host }
523
+ end
524
+
525
+ def demand_request(request, url, agent_id, deps)
526
+ out = { "url" => url.to_s, "agent_id" => agent_id, "user_agent" => request.headers.get("user-agent") || "",
527
+ "signals" => signals(request, deps, agent_id) }
528
+ sig = signature(request, url)
529
+ out["signature"] = sig if sig
530
+ out
531
+ end
532
+
533
+ # X- names the network can't add. Servers fold `_` and `-` into the same variable (WSGI, Rack, PHP), so names with `_`
534
+ # are refused outright. The rest steer routing, the method, middleware, or who the origin thinks the caller is.
535
+ DENIED_X_PREFIXES = %w[x-forwarded x-original x-rewrite x-middleware x-real x-client x-cluster-client x-originating x-remote x-host x-http-method x-method].freeze
536
+ DENIED_X_SUFFIXES = %w[-override -ip -user -host -url -uri -path -proto -scheme -port -for].freeze
537
+ HEADER_TOKEN = /\A[!#$%&'*+\-.^_`|~0-9A-Za-z]+\z/.freeze
538
+
539
+ # Header names the network may add to a release: its paywall key, credentials for the page, and custom X- headers. Nothing else.
540
+ def allowed_release_header?(name)
541
+ return false if name.include?("_")
542
+
543
+ n = name.downcase
544
+ return true if %w[cookie authorization constile-paywall-key].include?(n)
545
+ return false unless n.start_with?("x-")
546
+
547
+ DENIED_X_PREFIXES.none? { |p| n.start_with?(p) } && DENIED_X_SUFFIXES.none? { |x| n.end_with?(x) }
548
+ end
549
+
550
+ # Whether a header can carry this name and value, as the Fetch API's Headers decides.
551
+ def header_carries?(name, value)
552
+ return false unless name.valid_encoding? && value.valid_encoding?
553
+ return false unless HEADER_TOKEN.match?(name)
554
+ return false if value.match?(/[\0\r\n]/)
555
+
556
+ value.encode(Encoding::UTF_8).each_char.all? { |c| c.ord <= 0xFF }
557
+ rescue ArgumentError, EncodingError
558
+ false
559
+ end
560
+
561
+ # The URL to fetch: release_url only on the page's own scheme, host, and port, with no username or password.
562
+ def release_target(url, release_url)
563
+ page = URL.parse(url)
564
+ return page.to_s unless release_url.is_a?(String)
565
+
566
+ wanted = URL.parse(release_url)
567
+ # `host` includes a non-default port, so another port is another host. Userinfo is never accepted.
568
+ return wanted.to_s if wanted.scheme == page.scheme && wanted.host == page.host && !wanted.credentials?
569
+
570
+ page.to_s
571
+ rescue ArgumentError
572
+ URL.parse(url).to_s
573
+ end
574
+
575
+ # Fetches the article for an agent that has access. Never forwards the agent's headers to the origin.
576
+ # For a paywalled page the network's answer adds release instructions: a `release_url` (used only on the page's own
577
+ # scheme, host, and port) and `release_headers` (only the allowed names; the rest are dropped without a trace, and no
578
+ # name or value is logged).
579
+ def release_article(deps, url, instructions = nil)
580
+ instructions = {} unless instructions.is_a?(Hash)
581
+ sent = { "accept" => RELEASE_ACCEPT, "user-agent" => "#{GATE_VERSION} (release)" }
582
+ extra = instructions["release_headers"]
583
+ if extra.is_a?(Hash)
584
+ extra.each do |name, value|
585
+ next unless name.is_a?(String) && value.is_a?(String) && header_carries?(name, value) && allowed_release_header?(name)
586
+
587
+ sent[name.downcase] = value
588
+ end
589
+ end
590
+ res = deps.origin.call(Request.new(
591
+ request_method: "GET",
592
+ url: release_target(url, instructions["release_url"]),
593
+ headers: Headers.new(sent)
594
+ ))
595
+ replaced = SPONSORED_HEADERS.keys + ["set-cookie"]
596
+ headers = {}
597
+ res.headers.each { |k, v| headers[k] = v unless replaced.include?(k.to_s.downcase) }
598
+ Response.new(res.status, headers.merge(SPONSORED_HEADERS), res.body)
599
+ end
600
+
601
+ def log(deps, message, error = nil)
602
+ text = error ? "#{message}: #{error.class}: #{error.message}" : message
603
+ logger = deps.logger
604
+ if logger.respond_to?(:warn)
605
+ logger.warn(text)
606
+ elsif logger.respond_to?(:call)
607
+ logger.call(text)
608
+ else
609
+ Kernel.warn(text)
610
+ end
611
+ end
612
+ end