constile_gate 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +226 -0
- data/constile_gate.gemspec +21 -0
- data/lib/constile_gate/agents.rb +50 -0
- data/lib/constile_gate/charset.rb +427 -0
- data/lib/constile_gate/charset_tables.rb +284 -0
- data/lib/constile_gate/gate.rb +612 -0
- data/lib/constile_gate/http.rb +235 -0
- data/lib/constile_gate/js.rb +99 -0
- data/lib/constile_gate/page_meta.rb +83 -0
- data/lib/constile_gate/rack.rb +371 -0
- data/lib/constile_gate/railtie.rb +17 -0
- data/lib/constile_gate/url.rb +208 -0
- data/lib/constile_gate/version.rb +5 -0
- data/lib/constile_gate.rb +6 -0
- metadata +62 -0
|
@@ -0,0 +1,371 @@
|
|
|
1
|
+
require "ipaddr"
|
|
2
|
+
require "stringio"
|
|
3
|
+
require "constile_gate/gate"
|
|
4
|
+
|
|
5
|
+
module ConstileGate
|
|
6
|
+
# Rack middleware that runs the gate in front of a Rack app: Rails, Sinatra, or
|
|
7
|
+
# any other. It gets the page by calling the app in-process,
|
|
8
|
+
# so the gate never fetches its own site over HTTP and is never asked twice.
|
|
9
|
+
#
|
|
10
|
+
# It reads the whole response only for a candidate agent's request to an
|
|
11
|
+
# eligible page; every other response goes back untouched, body and all.
|
|
12
|
+
#
|
|
13
|
+
# use ConstileGate::Rack # settings from CONSTILE_SITE_ID, CONSTILE_SITE_SECRET, CONSTILE_NETWORK_URL
|
|
14
|
+
# use ConstileGate::Rack, trusted_proxies: ["10.0.0.0/8"]
|
|
15
|
+
#
|
|
16
|
+
# Options (each overrides its environment variable):
|
|
17
|
+
# site_id, site_secret, network_url the site's settings
|
|
18
|
+
# timeout seconds for each network step (CONSTILE_TIMEOUT, default 5)
|
|
19
|
+
# config_retry after a failed config fetch, seconds to pass everything through before asking again
|
|
20
|
+
# (CONSTILE_CONFIG_RETRY, default 30)
|
|
21
|
+
# trusted_proxies addresses or CIDR ranges whose X-Forwarded-For is believed (CONSTILE_TRUSTED_PROXIES, comma-separated)
|
|
22
|
+
# platform the install's name in the seller file and to the network (default "rack")
|
|
23
|
+
# logger an object with #warn, or a callable (default: Kernel#warn)
|
|
24
|
+
# enabled false passes everything through without a warning
|
|
25
|
+
#
|
|
26
|
+
# CONSTILE_DISABLED (1, true, yes, or on) turns the gate off too, read on every request.
|
|
27
|
+
class Rack
|
|
28
|
+
# Env keys that belong to one request, dropped from the env a release is built on.
|
|
29
|
+
PER_REQUEST_KEYS = %w[
|
|
30
|
+
CONTENT_TYPE CONTENT_LENGTH REQUEST_METHOD SCRIPT_NAME PATH_INFO QUERY_STRING REQUEST_URI REQUEST_PATH
|
|
31
|
+
ORIGINAL_FULLPATH ORIGINAL_SCRIPT_NAME HTTPS SERVER_NAME SERVER_PORT
|
|
32
|
+
rack.url_scheme rack.input rack.hijack rack.hijack? rack.hijack_io rack.response_finished rack.early_hints
|
|
33
|
+
rack.tempfiles rack.methodoverride.original_method action_dispatch.cookies action_dispatch.remote_ip warden
|
|
34
|
+
].freeze
|
|
35
|
+
PER_REQUEST_PREFIXES = %w[HTTP_ rack.request. rack.session rack.lint action_dispatch.request].freeze
|
|
36
|
+
SCHEMES = { "http" => "http", "https" => "https", "ws" => "http", "wss" => "https" }.freeze
|
|
37
|
+
MISSING = "CONSTILE_SITE_ID, CONSTILE_SITE_SECRET, or CONSTILE_NETWORK_URL is missing; passing everything through".freeze
|
|
38
|
+
|
|
39
|
+
def initialize(app, *option_hashes)
|
|
40
|
+
@app = app
|
|
41
|
+
opts = option_hashes.each_with_object({}) { |h, all| h.to_h.each { |k, v| all[k.to_sym] = v } }
|
|
42
|
+
@site_id = setting(opts[:site_id], "CONSTILE_SITE_ID")
|
|
43
|
+
@secret = setting(opts[:site_secret], "CONSTILE_SITE_SECRET")
|
|
44
|
+
@network_url = setting(opts[:network_url], "CONSTILE_NETWORK_URL")
|
|
45
|
+
@platform = (opts[:platform] || "rack").to_s
|
|
46
|
+
@logger = opts[:logger]
|
|
47
|
+
timeout = Float(setting(opts[:timeout], "CONSTILE_TIMEOUT") || 5)
|
|
48
|
+
retry_s = setting(opts[:config_retry], "CONSTILE_CONFIG_RETRY")
|
|
49
|
+
@config_retry_s = retry_s && Float(retry_s)
|
|
50
|
+
@network = opts[:network] || HTTPClient.new(timeout: timeout)
|
|
51
|
+
@background = opts[:background] || Background.shared
|
|
52
|
+
@trusted = TrustedProxies.new(opts.key?(:trusted_proxies) ? opts[:trusted_proxies] : ENV["CONSTILE_TRUSTED_PROXIES"])
|
|
53
|
+
@consume_each = !(defined?(::Rack::RELEASE) && ::Rack::RELEASE.to_s.to_i >= 3)
|
|
54
|
+
@enabled = opts[:enabled] != false
|
|
55
|
+
return unless @enabled && !(@site_id && @secret && @network_url)
|
|
56
|
+
|
|
57
|
+
@enabled = false
|
|
58
|
+
ConstileGate.log(logger_deps, "#{GATE_VERSION}: #{MISSING}")
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def call(env)
|
|
62
|
+
return @app.call(env) unless @enabled
|
|
63
|
+
# Off: the app as if the gate weren't there.
|
|
64
|
+
return @app.call(env) if ConstileGate.on?(ENV["CONSTILE_DISABLED"])
|
|
65
|
+
|
|
66
|
+
url = begin
|
|
67
|
+
page_url(env)
|
|
68
|
+
rescue ArgumentError
|
|
69
|
+
nil
|
|
70
|
+
end
|
|
71
|
+
# A request the gate can't read (a malformed Host header, say) goes to the app as if the gate weren't there.
|
|
72
|
+
return @app.call(env) unless url
|
|
73
|
+
|
|
74
|
+
ip = visitor_address(env)
|
|
75
|
+
attempt = Attempt.new
|
|
76
|
+
deps = Deps.new(
|
|
77
|
+
site_id: @site_id, secret: @secret, network_url: @network_url, network: @network, platform: @platform,
|
|
78
|
+
config_retry_s: @config_retry_s,
|
|
79
|
+
logger: @logger, origin: ->(req) { fetch(env, req, attempt) }, page_origin: ->(req) { fetch_page(env, req, attempt) },
|
|
80
|
+
wait_until: ->(job) { @background.push(job) },
|
|
81
|
+
signals: { asn: nil, as_org: nil, country: nil, ip_prefix: ConstileGate.ip_prefix(ip), ip: ip }
|
|
82
|
+
)
|
|
83
|
+
request = Request.new(request_method: env["REQUEST_METHOD"].to_s, url: url, headers: EnvHeaders.new(env), env: env)
|
|
84
|
+
begin
|
|
85
|
+
res = ConstileGate.handle_gate(request, deps)
|
|
86
|
+
rescue StandardError => e
|
|
87
|
+
# Errors from the app go on unchanged, as they would without the gate.
|
|
88
|
+
raise if e.equal?(attempt.app_error) || attempt.page&.read_failed
|
|
89
|
+
|
|
90
|
+
return failed(e, env, url, attempt)
|
|
91
|
+
end
|
|
92
|
+
respond(res, env)
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# The secret never prints.
|
|
96
|
+
def inspect
|
|
97
|
+
"#<#{self.class.name} site_id=#{@site_id.inspect}, network_url=#{@network_url.inspect}, platform=#{@platform.inspect}, " \
|
|
98
|
+
"enabled=#{@enabled}>"
|
|
99
|
+
end
|
|
100
|
+
alias_method :to_s, :inspect
|
|
101
|
+
|
|
102
|
+
private
|
|
103
|
+
|
|
104
|
+
def setting(value, name)
|
|
105
|
+
v = value.nil? ? ENV[name] : value
|
|
106
|
+
v.nil? || v.to_s.strip.empty? ? nil : v.to_s
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def logger_deps
|
|
110
|
+
Deps.new(logger: @logger)
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# What the app has done for one request, so a gate error can fall back on it.
|
|
114
|
+
Attempt = Struct.new(:page, :app_error)
|
|
115
|
+
|
|
116
|
+
# The origin: the request itself goes to the app as it came; a release is a fresh
|
|
117
|
+
# GET built from the server's env, without the agent's headers.
|
|
118
|
+
def fetch(env, req, attempt)
|
|
119
|
+
target = req.env || release_env(env, req)
|
|
120
|
+
begin
|
|
121
|
+
status, headers, body = @app.call(target)
|
|
122
|
+
rescue StandardError => e
|
|
123
|
+
attempt.app_error = e
|
|
124
|
+
raise
|
|
125
|
+
end
|
|
126
|
+
res = AppResponse.new(status, headers, body, env: target, consume_each: @consume_each)
|
|
127
|
+
attempt.page = res if target.equal?(env)
|
|
128
|
+
res
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
# An unexpected error in the gate fails open, as a network outage does: the
|
|
132
|
+
# visitor gets the app's answer to the page, or the app is asked now. The gate's
|
|
133
|
+
# own routes answer 503 instead: the app doesn't serve them, and a callback never
|
|
134
|
+
# falls back to the page, least of all a release.
|
|
135
|
+
def failed(error, env, url, attempt)
|
|
136
|
+
ConstileGate.log(logger_deps, "constile gate: unexpected error, passing through", error)
|
|
137
|
+
return respond(ConstileGate.json({ "error" => UNAVAILABLE }, 503), env) if ConstileGate.gate_route?(url.pathname)
|
|
138
|
+
return attempt.page.to_rack if attempt.page
|
|
139
|
+
|
|
140
|
+
@app.call(env)
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# The page an agent's offer is for. Without Accept-Encoding, the app (and any
|
|
144
|
+
# Rack::Deflater inside the gate) answers uncompressed. Rack 3 bars middleware from
|
|
145
|
+
# reading a compressed stream, so a compressed page could get no offer.
|
|
146
|
+
def fetch_page(env, req, attempt)
|
|
147
|
+
env.delete("HTTP_ACCEPT_ENCODING")
|
|
148
|
+
fetch(env, req, attempt)
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def respond(res, env)
|
|
152
|
+
# The app's own answer to this request goes back exactly as the app gave it.
|
|
153
|
+
return [res.status, res.headers, res.body] if res.is_a?(AppResponse) && res.env.equal?(env)
|
|
154
|
+
|
|
155
|
+
body = res.body
|
|
156
|
+
if env["REQUEST_METHOD"] == "HEAD"
|
|
157
|
+
body.close if body.respond_to?(:close)
|
|
158
|
+
body = []
|
|
159
|
+
end
|
|
160
|
+
[res.status.to_i, res.headers, body]
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# The URL the visitor asked for: the scheme and host (from X-Forwarded-Proto and
|
|
164
|
+
# X-Forwarded-Host only when the peer is a trusted proxy), and the full path,
|
|
165
|
+
# SCRIPT_NAME and PATH_INFO together.
|
|
166
|
+
def page_url(env)
|
|
167
|
+
proxied = @trusted.trusted?(env["REMOTE_ADDR"].to_s.strip)
|
|
168
|
+
host = (proxied && last_value(env["HTTP_X_FORWARDED_HOST"])) || env["HTTP_HOST"]
|
|
169
|
+
if host.nil? || host.empty?
|
|
170
|
+
port = env["SERVER_PORT"].to_s
|
|
171
|
+
host = port.empty? ? env["SERVER_NAME"].to_s : "#{env["SERVER_NAME"]}:#{port}"
|
|
172
|
+
end
|
|
173
|
+
path = "#{env["SCRIPT_NAME"]}#{env["PATH_INFO"]}".b
|
|
174
|
+
path = "/" if path.empty?
|
|
175
|
+
query = env["QUERY_STRING"].to_s
|
|
176
|
+
URL.parse("#{scheme(env, proxied)}://#{host.b}#{path}#{query.empty? ? "" : "?#{query.b}"}".b)
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
# The server's scheme, or, from a trusted proxy, the scheme it reports, as Rack::Request reads it.
|
|
180
|
+
def scheme(env, proxied)
|
|
181
|
+
return "https" if env["HTTPS"] == "on" || (proxied && env["HTTP_X_FORWARDED_SSL"] == "on")
|
|
182
|
+
|
|
183
|
+
if proxied
|
|
184
|
+
%w[HTTP_X_FORWARDED_PROTO HTTP_X_FORWARDED_SCHEME].each do |key|
|
|
185
|
+
env[key].to_s.split(",").map(&:strip).reverse_each { |s| return SCHEMES[s] if SCHEMES[s] }
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
SCHEMES[env["rack.url_scheme"].to_s] || "http"
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
def last_value(header)
|
|
192
|
+
v = header.to_s.split(",").map(&:strip).reject(&:empty?).last
|
|
193
|
+
v && !v.empty? ? v : nil
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
def visitor_address(env)
|
|
197
|
+
addr = @trusted.client_address(env["REMOTE_ADDR"].to_s.strip, env["HTTP_X_FORWARDED_FOR"])
|
|
198
|
+
return nil if addr.nil? || addr.empty?
|
|
199
|
+
|
|
200
|
+
addr.sub(/\A::ffff:(\d+\.\d+\.\d+\.\d+)\z/i, '\1')
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
def release_env(env, req)
|
|
204
|
+
url = URL.parse(req.url)
|
|
205
|
+
out = env.reject { |k, _| PER_REQUEST_KEYS.include?(k) || PER_REQUEST_PREFIXES.any? { |p| k.start_with?(p) } }
|
|
206
|
+
out["REQUEST_METHOD"] = req.request_method
|
|
207
|
+
out["SCRIPT_NAME"] = ""
|
|
208
|
+
out["PATH_INFO"] = url.pathname.b
|
|
209
|
+
out["QUERY_STRING"] = url.query.to_s.b
|
|
210
|
+
out["SERVER_NAME"] = url.hostname.b
|
|
211
|
+
out["SERVER_PORT"] = (url.port || URL::DEFAULT_PORTS[url.scheme]).to_s
|
|
212
|
+
out["SERVER_PROTOCOL"] ||= "HTTP/1.1"
|
|
213
|
+
out["HTTP_HOST"] = url.host.b
|
|
214
|
+
out["rack.url_scheme"] = url.scheme
|
|
215
|
+
out["HTTPS"] = "on" if url.scheme == "https"
|
|
216
|
+
out["rack.input"] = StringIO.new(String.new(encoding: Encoding::BINARY))
|
|
217
|
+
req.headers.each { |name, value| out[EnvHeaders.key(name)] = value.b }
|
|
218
|
+
out
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
# A response from the wrapped app, remembering the env it answered.
|
|
222
|
+
class AppResponse < Response
|
|
223
|
+
attr_reader :env, :read_failed
|
|
224
|
+
|
|
225
|
+
def initialize(status, headers, body, env:, consume_each:)
|
|
226
|
+
super(status, headers, body, consume_each: consume_each)
|
|
227
|
+
@env = env
|
|
228
|
+
@read_failed = false
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
def read_body
|
|
232
|
+
super
|
|
233
|
+
rescue StandardError
|
|
234
|
+
@read_failed = true
|
|
235
|
+
raise
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
# The response for Rack: the bytes already read, or the body untouched.
|
|
239
|
+
def to_rack
|
|
240
|
+
raw = defined?(@raw) ? @raw : nil
|
|
241
|
+
[status, headers, raw ? [raw] : body]
|
|
242
|
+
end
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
# Request headers read from a Rack env, the way a fetch Headers object holds them.
|
|
246
|
+
class EnvHeaders
|
|
247
|
+
def self.key(name)
|
|
248
|
+
n = name.to_s.downcase
|
|
249
|
+
return "CONTENT_TYPE" if n == "content-type"
|
|
250
|
+
return "CONTENT_LENGTH" if n == "content-length"
|
|
251
|
+
|
|
252
|
+
"HTTP_#{n.upcase.tr("-", "_")}"
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
def initialize(env)
|
|
256
|
+
@env = env
|
|
257
|
+
end
|
|
258
|
+
|
|
259
|
+
def get(name)
|
|
260
|
+
v = @env[EnvHeaders.key(name)]
|
|
261
|
+
v.nil? ? nil : Headers.normalize(v)
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
def has?(name)
|
|
265
|
+
@env.key?(EnvHeaders.key(name))
|
|
266
|
+
end
|
|
267
|
+
|
|
268
|
+
# The env holds the visitor's credentials, so it never prints.
|
|
269
|
+
def inspect
|
|
270
|
+
"#<#{self.class.name}>"
|
|
271
|
+
end
|
|
272
|
+
alias_method :to_s, :inspect
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
# The proxies whose X-Forwarded-For the publisher trusts. With none, the visitor's
|
|
276
|
+
# address is REMOTE_ADDR, so no client can claim another address.
|
|
277
|
+
class TrustedProxies
|
|
278
|
+
def initialize(list)
|
|
279
|
+
entries = list.is_a?(String) ? list.split(",") : Array(list)
|
|
280
|
+
@ranges = entries.map { |e| e.is_a?(IPAddr) ? e : e.to_s.strip }.reject { |e| e == "" }
|
|
281
|
+
.map { |e| e.is_a?(IPAddr) ? e : IPAddr.new(e) }
|
|
282
|
+
end
|
|
283
|
+
|
|
284
|
+
# Walks X-Forwarded-For from the right, past trusted proxies, to the first address
|
|
285
|
+
# that isn't one. A malformed hop stops the walk at the last trusted proxy.
|
|
286
|
+
def client_address(remote, forwarded_for)
|
|
287
|
+
return remote if @ranges.empty? || forwarded_for.nil? || !trusted?(remote)
|
|
288
|
+
|
|
289
|
+
addr = remote
|
|
290
|
+
forwarded_for.to_s.split(",").map(&:strip).reverse_each do |hop|
|
|
291
|
+
hop = TrustedProxies.address(hop)
|
|
292
|
+
break unless hop
|
|
293
|
+
|
|
294
|
+
addr = hop
|
|
295
|
+
break unless trusted?(hop)
|
|
296
|
+
end
|
|
297
|
+
addr
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
def trusted?(addr)
|
|
301
|
+
ip = parse(addr)
|
|
302
|
+
!ip.nil? && @ranges.any? { |r| r.family == ip.family && r.include?(ip) }
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
# The address in a hop, without a port ("203.0.113.9:443", "[2001:db8::1]:443"); nil when it isn't one.
|
|
306
|
+
def self.address(hop)
|
|
307
|
+
a = hop.sub(/\A\[(.+)\](?::\d+)?\z/, '\1').sub(/\A(\d+\.\d+\.\d+\.\d+):\d+\z/, '\1')
|
|
308
|
+
IPAddr.new(a)
|
|
309
|
+
a
|
|
310
|
+
rescue IPAddr::Error, ArgumentError
|
|
311
|
+
nil
|
|
312
|
+
end
|
|
313
|
+
|
|
314
|
+
private
|
|
315
|
+
|
|
316
|
+
def parse(addr)
|
|
317
|
+
IPAddr.new(addr.to_s)
|
|
318
|
+
rescue IPAddr::Error, ArgumentError
|
|
319
|
+
nil
|
|
320
|
+
end
|
|
321
|
+
end
|
|
322
|
+
|
|
323
|
+
# Runs demand reports after the response, on one worker thread per process with a
|
|
324
|
+
# bounded queue, so a slow network never holds up a page. Reports beyond the queue
|
|
325
|
+
# are dropped: they're counts, not money.
|
|
326
|
+
class Background
|
|
327
|
+
LIMIT = 100
|
|
328
|
+
|
|
329
|
+
def self.shared
|
|
330
|
+
SHARED
|
|
331
|
+
end
|
|
332
|
+
|
|
333
|
+
def initialize
|
|
334
|
+
@mutex = Mutex.new
|
|
335
|
+
@queue = nil
|
|
336
|
+
@thread = nil
|
|
337
|
+
@pid = nil
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
def push(job)
|
|
341
|
+
@mutex.synchronize do
|
|
342
|
+
start unless @pid == Process.pid && @thread&.alive?
|
|
343
|
+
return false if @queue.size >= LIMIT
|
|
344
|
+
|
|
345
|
+
@queue << job
|
|
346
|
+
end
|
|
347
|
+
true
|
|
348
|
+
end
|
|
349
|
+
|
|
350
|
+
private
|
|
351
|
+
|
|
352
|
+
def start
|
|
353
|
+
@pid = Process.pid
|
|
354
|
+
@queue = Queue.new
|
|
355
|
+
queue = @queue
|
|
356
|
+
@thread = Thread.new do
|
|
357
|
+
loop do
|
|
358
|
+
job = queue.pop
|
|
359
|
+
begin
|
|
360
|
+
job.call
|
|
361
|
+
rescue StandardError
|
|
362
|
+
nil # each job logs its own failure
|
|
363
|
+
end
|
|
364
|
+
end
|
|
365
|
+
end
|
|
366
|
+
end
|
|
367
|
+
|
|
368
|
+
SHARED = new
|
|
369
|
+
end
|
|
370
|
+
end
|
|
371
|
+
end
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
module ConstileGate
|
|
2
|
+
# In a Rails app, puts the gate first in the middleware stack, in front of
|
|
3
|
+
# everything else, with "rails" as its platform. Settings come from CONSTILE_SITE_ID,
|
|
4
|
+
# CONSTILE_SITE_SECRET, and CONSTILE_NETWORK_URL, or from config.constile_gate (site_id,
|
|
5
|
+
# site_secret, network_url, timeout, trusted_proxies, enabled), which the
|
|
6
|
+
# middleware reads when Rails builds the stack, after config/initializers.
|
|
7
|
+
class Railtie < ::Rails::Railtie
|
|
8
|
+
config.constile_gate = ActiveSupport::OrderedOptions.new
|
|
9
|
+
|
|
10
|
+
initializer "constile_gate.middleware" do |app|
|
|
11
|
+
options = app.config.constile_gate
|
|
12
|
+
options.platform ||= "rails"
|
|
13
|
+
options.logger ||= Rails.logger
|
|
14
|
+
app.config.middleware.insert_before 0, ConstileGate::Rack, options
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
require "ipaddr"
|
|
2
|
+
|
|
3
|
+
module ConstileGate
|
|
4
|
+
# An absolute http or https URL, parsed and serialized the way the WHATWG URL
|
|
5
|
+
# standard (JavaScript's URL) does for the parts the gate uses: the scheme, host,
|
|
6
|
+
# path, and query. Hosts with non-ASCII characters are refused, because the
|
|
7
|
+
# standard's IDNA processing isn't ported.
|
|
8
|
+
class URL
|
|
9
|
+
DEFAULT_PORTS = { "http" => 80, "https" => 443 }.freeze
|
|
10
|
+
SHAPE = %r{\A([A-Za-z][A-Za-z0-9+.\-]*):[/\\]*([^/\\?#]*)([^?#]*)(?:\?([^#]*))?(?:#(.*))?\z}m.freeze
|
|
11
|
+
# Code points the standard forbids in a domain, after percent-decoding.
|
|
12
|
+
FORBIDDEN_DOMAIN = %r{[\x00-\x20#%/:<>?@\[\\\]^|\x7F]}n.freeze
|
|
13
|
+
# Bytes percent-encoded in each component, beyond C0 controls and bytes above 0x7E.
|
|
14
|
+
PATH_SET = " \"#<>?`{}".bytes.freeze
|
|
15
|
+
QUERY_SET = " \"#<>'".bytes.freeze
|
|
16
|
+
FRAGMENT_SET = " \"<>`".bytes.freeze
|
|
17
|
+
|
|
18
|
+
attr_reader :scheme, :hostname, :port, :pathname, :query, :fragment
|
|
19
|
+
|
|
20
|
+
def self.parse(input)
|
|
21
|
+
return input if input.is_a?(URL)
|
|
22
|
+
|
|
23
|
+
new(input.to_s)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def initialize(input)
|
|
27
|
+
s = input.b.delete("\t\n\r").sub(/\A[\x00-\x20]+/n, "").sub(/[\x00-\x20]+\z/n, "")
|
|
28
|
+
m = SHAPE.match(s) or raise ArgumentError, "Invalid URL"
|
|
29
|
+
@scheme = m[1].downcase
|
|
30
|
+
raise ArgumentError, "Unsupported scheme #{@scheme}" unless DEFAULT_PORTS.key?(@scheme)
|
|
31
|
+
|
|
32
|
+
authority = m[2]
|
|
33
|
+
@credentials = false
|
|
34
|
+
if authority.include?("@")
|
|
35
|
+
userinfo = authority[0...authority.rindex("@")]
|
|
36
|
+
# A username or password is anything but an empty userinfo or a lone colon.
|
|
37
|
+
@credentials = !["", ":"].include?(userinfo)
|
|
38
|
+
authority = authority[(authority.rindex("@") + 1)..-1]
|
|
39
|
+
end
|
|
40
|
+
@hostname, @port = parse_host_and_port(authority)
|
|
41
|
+
@pathname = normalize_path(m[3])
|
|
42
|
+
@query = m[4] && encode(m[4], QUERY_SET)
|
|
43
|
+
@fragment = m[5] && encode(m[5], FRAGMENT_SET)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# Whether the input carried a username or password (URL#username or URL#password is not empty).
|
|
47
|
+
def credentials?
|
|
48
|
+
@credentials
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# The host with its port, when the port isn't the scheme's default (URL#host).
|
|
52
|
+
def host
|
|
53
|
+
@port ? "#{@hostname}:#{@port}" : @hostname
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def origin
|
|
57
|
+
"#{@scheme}://#{host}"
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# URL#search: "" for an empty or missing query.
|
|
61
|
+
def search
|
|
62
|
+
@query.nil? || @query.empty? ? "" : "?#{@query}"
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def to_s
|
|
66
|
+
out = +"#{origin}#{@pathname}"
|
|
67
|
+
out << "?#{@query}" if @query
|
|
68
|
+
out << "##{@fragment}" if @fragment
|
|
69
|
+
out
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# URLSearchParams#get: the first value for the name, or nil.
|
|
73
|
+
def search_param(name)
|
|
74
|
+
return nil if @query.nil?
|
|
75
|
+
|
|
76
|
+
@query.split("&").each do |pair|
|
|
77
|
+
next if pair.empty?
|
|
78
|
+
|
|
79
|
+
k, v = pair.split("=", 2)
|
|
80
|
+
return form_decode(v || "") if form_decode(k) == name
|
|
81
|
+
end
|
|
82
|
+
nil
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
private
|
|
86
|
+
|
|
87
|
+
def parse_host_and_port(authority)
|
|
88
|
+
if authority.start_with?("[")
|
|
89
|
+
close = authority.index("]") or raise ArgumentError, "Invalid IPv6 host"
|
|
90
|
+
host = "[#{serialize_ipv6(authority[1...close])}]"
|
|
91
|
+
rest = authority[(close + 1)..-1]
|
|
92
|
+
raise ArgumentError, "Invalid host" unless rest.empty? || rest.start_with?(":")
|
|
93
|
+
|
|
94
|
+
port = rest.empty? ? "" : rest[1..-1]
|
|
95
|
+
else
|
|
96
|
+
host, port = authority.include?(":") ? authority.split(":", 2) : [authority, ""]
|
|
97
|
+
host = parse_domain(host)
|
|
98
|
+
end
|
|
99
|
+
[host, parse_port(port)]
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def parse_domain(raw)
|
|
103
|
+
host = raw.gsub(/%([0-9A-Fa-f]{2})/n) { Regexp.last_match(1).hex.chr }
|
|
104
|
+
raise ArgumentError, "Empty host" if host.empty?
|
|
105
|
+
raise ArgumentError, "Non-ASCII host" unless host.ascii_only?
|
|
106
|
+
|
|
107
|
+
host = host.downcase
|
|
108
|
+
raise ArgumentError, "Forbidden host code point" if host.match?(FORBIDDEN_DOMAIN)
|
|
109
|
+
|
|
110
|
+
ipv4(host) || host.force_encoding(Encoding::UTF_8)
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# A host whose last label is a number is an IPv4 address, written in any of the
|
|
114
|
+
# standard's forms (127.1, 0x7f.0.0.1, 0177.0.0.1).
|
|
115
|
+
def ipv4(host)
|
|
116
|
+
labels = host.split(".", -1)
|
|
117
|
+
labels.pop if labels.size > 1 && labels.last.empty?
|
|
118
|
+
return nil unless labels.last =~ /\A(?:0[xX][0-9A-Fa-f]*|[0-9]+)\z/
|
|
119
|
+
|
|
120
|
+
raise ArgumentError, "Invalid IPv4 host" if labels.size > 4
|
|
121
|
+
|
|
122
|
+
numbers = labels.map do |part|
|
|
123
|
+
case part
|
|
124
|
+
when /\A0[xX]([0-9A-Fa-f]*)\z/ then Regexp.last_match(1).to_i(16)
|
|
125
|
+
when /\A0[0-9]+\z/
|
|
126
|
+
raise ArgumentError, "Invalid IPv4 host" unless part.match?(/\A[0-7]+\z/)
|
|
127
|
+
|
|
128
|
+
part.to_i(8)
|
|
129
|
+
when /\A[0-9]+\z/ then part.to_i
|
|
130
|
+
else raise ArgumentError, "Invalid IPv4 host"
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
last = numbers.pop
|
|
134
|
+
raise ArgumentError, "Invalid IPv4 host" if numbers.any? { |n| n > 255 } || last >= 256**(4 - numbers.size)
|
|
135
|
+
|
|
136
|
+
value = numbers.each_with_index.sum { |n, i| n * 256**(3 - i) } + last
|
|
137
|
+
[24, 16, 8, 0].map { |shift| (value >> shift) & 255 }.join(".")
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def serialize_ipv6(raw)
|
|
141
|
+
addr = IPAddr.new(raw)
|
|
142
|
+
raise ArgumentError, "Invalid IPv6 host" unless addr.ipv6?
|
|
143
|
+
|
|
144
|
+
pieces = addr.to_i.digits(65_536).reverse
|
|
145
|
+
pieces.unshift(0) while pieces.size < 8
|
|
146
|
+
# Compress the first longest run of two or more zero pieces.
|
|
147
|
+
best_start = best_len = nil
|
|
148
|
+
i = 0
|
|
149
|
+
while i < 8
|
|
150
|
+
if pieces[i].zero?
|
|
151
|
+
j = i
|
|
152
|
+
j += 1 while j < 8 && pieces[j].zero?
|
|
153
|
+
best_start, best_len = i, j - i if j - i > 1 && (best_len.nil? || j - i > best_len)
|
|
154
|
+
i = j
|
|
155
|
+
else
|
|
156
|
+
i += 1
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
hex = pieces.map { |p| p.to_s(16) }
|
|
160
|
+
return hex.join(":") unless best_start
|
|
161
|
+
|
|
162
|
+
"#{hex[0...best_start].join(":")}::#{hex[(best_start + best_len)..-1].join(":")}"
|
|
163
|
+
rescue IPAddr::Error
|
|
164
|
+
raise ArgumentError, "Invalid IPv6 host"
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
def parse_port(raw)
|
|
168
|
+
return nil if raw.empty?
|
|
169
|
+
raise ArgumentError, "Invalid port" unless raw.match?(/\A[0-9]+\z/n)
|
|
170
|
+
|
|
171
|
+
port = raw.to_i
|
|
172
|
+
raise ArgumentError, "Invalid port" if port > 65_535
|
|
173
|
+
|
|
174
|
+
port == DEFAULT_PORTS[@scheme] ? nil : port
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def normalize_path(raw)
|
|
178
|
+
raw = raw.tr("\\", "/")
|
|
179
|
+
return "/" if raw.empty?
|
|
180
|
+
|
|
181
|
+
segments = raw[1..-1].split("/", -1)
|
|
182
|
+
out = []
|
|
183
|
+
segments.each_with_index do |seg, i|
|
|
184
|
+
last = i == segments.size - 1
|
|
185
|
+
if seg.match?(/\A(?:\.|%2[eE]){2}\z/n)
|
|
186
|
+
out.pop
|
|
187
|
+
out << "" if last
|
|
188
|
+
elsif seg.match?(/\A(?:\.|%2[eE])\z/n)
|
|
189
|
+
out << "" if last
|
|
190
|
+
else
|
|
191
|
+
out << encode(seg, PATH_SET)
|
|
192
|
+
end
|
|
193
|
+
end
|
|
194
|
+
"/#{out.join("/")}"
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
def encode(str, set)
|
|
198
|
+
out = String.new(encoding: Encoding::UTF_8)
|
|
199
|
+
str.each_byte { |b| out << (b < 0x20 || b > 0x7E || set.include?(b) ? format("%%%02X", b) : b.chr) }
|
|
200
|
+
out
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
# application/x-www-form-urlencoded decoding, lenient like URLSearchParams.
|
|
204
|
+
def form_decode(str)
|
|
205
|
+
str.tr("+", " ").b.gsub(/%([0-9A-Fa-f]{2})/n) { Regexp.last_match(1).hex.chr }.force_encoding(Encoding::UTF_8).scrub("\uFFFD")
|
|
206
|
+
end
|
|
207
|
+
end
|
|
208
|
+
end
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
# The Constile gate for Ruby: the gate (ConstileGate.handle_gate), Rack
|
|
2
|
+
# middleware (ConstileGate::Rack), and, in a Rails app, a Railtie that inserts it.
|
|
3
|
+
require "constile_gate/version"
|
|
4
|
+
require "constile_gate/gate"
|
|
5
|
+
require "constile_gate/rack"
|
|
6
|
+
require "constile_gate/railtie" if defined?(::Rails::Railtie)
|
metadata
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: constile_gate
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.6.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- CCC2C Inc.
|
|
8
|
+
autorequire:
|
|
9
|
+
bindir: bin
|
|
10
|
+
cert_chain: []
|
|
11
|
+
date: 2026-11-18 00:00:00.000000000 Z
|
|
12
|
+
dependencies: []
|
|
13
|
+
description: Serves the Constile offer to AI agents that fetch a page for a person,
|
|
14
|
+
forwards the offer's callbacks to the Constile network, and passes everyone else
|
|
15
|
+
through untouched.
|
|
16
|
+
email:
|
|
17
|
+
- hello@constile.app
|
|
18
|
+
executables: []
|
|
19
|
+
extensions: []
|
|
20
|
+
extra_rdoc_files: []
|
|
21
|
+
files:
|
|
22
|
+
- LICENSE
|
|
23
|
+
- README.md
|
|
24
|
+
- constile_gate.gemspec
|
|
25
|
+
- lib/constile_gate.rb
|
|
26
|
+
- lib/constile_gate/agents.rb
|
|
27
|
+
- lib/constile_gate/charset.rb
|
|
28
|
+
- lib/constile_gate/charset_tables.rb
|
|
29
|
+
- lib/constile_gate/gate.rb
|
|
30
|
+
- lib/constile_gate/http.rb
|
|
31
|
+
- lib/constile_gate/js.rb
|
|
32
|
+
- lib/constile_gate/page_meta.rb
|
|
33
|
+
- lib/constile_gate/rack.rb
|
|
34
|
+
- lib/constile_gate/railtie.rb
|
|
35
|
+
- lib/constile_gate/url.rb
|
|
36
|
+
- lib/constile_gate/version.rb
|
|
37
|
+
homepage: https://constile.app
|
|
38
|
+
licenses:
|
|
39
|
+
- MIT
|
|
40
|
+
metadata:
|
|
41
|
+
documentation_uri: https://constile.app/setup/rack
|
|
42
|
+
rubygems_mfa_required: 'true'
|
|
43
|
+
post_install_message:
|
|
44
|
+
rdoc_options: []
|
|
45
|
+
require_paths:
|
|
46
|
+
- lib
|
|
47
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
48
|
+
requirements:
|
|
49
|
+
- - ">="
|
|
50
|
+
- !ruby/object:Gem::Version
|
|
51
|
+
version: '2.7'
|
|
52
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
53
|
+
requirements:
|
|
54
|
+
- - ">="
|
|
55
|
+
- !ruby/object:Gem::Version
|
|
56
|
+
version: '0'
|
|
57
|
+
requirements: []
|
|
58
|
+
rubygems_version: 3.5.22
|
|
59
|
+
signing_key:
|
|
60
|
+
specification_version: 4
|
|
61
|
+
summary: The Constile gate as Rack middleware, for Rails, Sinatra, and any Rack app.
|
|
62
|
+
test_files: []
|