constile_gate 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +22 -6
- data/lib/constile_gate/agents.rb +18 -3
- data/lib/constile_gate/gate.rb +11 -3
- data/lib/constile_gate/version.rb +2 -2
- metadata +3 -6
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 0a141ab230b92095bf9c79e988dc90d0c91ee7e0c0233ae8d9f90640d58b53cc
|
|
4
|
+
data.tar.gz: 3db2ae742d4ff1a24fc5822ef0b10869161f0c9832b0e546bc72d68a4363f7ed
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 407bb764b81500c3a7292ed9f00b757c62abf09429fac728c1153e5ae78ac5d5b452f9389ea294f8f7bdfed911770f24fac4f0e51c7c3146dc7a0c20a7cb3752
|
|
7
|
+
data.tar.gz: a2e3027ec0ee3dfb682bc724e184a3b09505976279bfe78733756814c094a21bce4b9e41337733178e2674362e3dbda69fc0d73b97aa1fb822638f3493f6a424
|
data/README.md
CHANGED
|
@@ -25,18 +25,18 @@ The gate answers at the root of your domain, at `/.well-known/constile` and
|
|
|
25
25
|
Add it to your `Gemfile`, then run `bundle install`:
|
|
26
26
|
|
|
27
27
|
```ruby
|
|
28
|
-
gem "constile_gate", "~> 0.
|
|
28
|
+
gem "constile_gate", "~> 0.7"
|
|
29
29
|
```
|
|
30
30
|
|
|
31
31
|
If your app doesn't use Bundler, run `gem install constile_gate` on each server
|
|
32
32
|
instead.
|
|
33
33
|
|
|
34
34
|
To use the copy from your site page instead, download
|
|
35
|
-
`constile_gate-0.
|
|
36
|
-
`gem unpack constile_gate-0.
|
|
35
|
+
`constile_gate-0.7.0.gem`, unpack it into your app with
|
|
36
|
+
`gem unpack constile_gate-0.7.0.gem --target vendor/gems`, point your
|
|
37
37
|
`Gemfile` at it with
|
|
38
|
-
`gem "constile_gate", path: "vendor/gems/constile_gate-0.
|
|
39
|
-
`vendor/gems/constile_gate-0.
|
|
38
|
+
`gem "constile_gate", path: "vendor/gems/constile_gate-0.7.0"`, and commit
|
|
39
|
+
`vendor/gems/constile_gate-0.7.0` so your deploys get it too.
|
|
40
40
|
|
|
41
41
|
## Set your site's settings
|
|
42
42
|
|
|
@@ -140,6 +140,22 @@ site's page in the dashboard and choose a content API, a paywall rule, or a
|
|
|
140
140
|
Constile login. A paywalled page gets offers only after the method works and
|
|
141
141
|
the gate is 0.6.0 or later.
|
|
142
142
|
|
|
143
|
+
## Scripts that pose as a browser
|
|
144
|
+
|
|
145
|
+
Some scripts send a browser's user agent so they get your pages as a person
|
|
146
|
+
would. Gate 0.7.0 and later can send them an ad instead: on your site's page
|
|
147
|
+
in the dashboard, under **Settings**, set **Scripts that pose as a browser**
|
|
148
|
+
to **Send them an ad**. The gate then takes a request with a browser's user
|
|
149
|
+
agent for a script when it sends neither `Accept-Language` nor
|
|
150
|
+
`Sec-Fetch-Mode`, headers browsers always send, and has no `constile_browser`
|
|
151
|
+
cookie. These ads count toward your site's daily limit on earnings from agents
|
|
152
|
+
nothing could verify. If a person's browser is ever taken for a script, the ad
|
|
153
|
+
page says the page loads in a moment, sets the `constile_browser` cookie, and
|
|
154
|
+
reloads, so the person gets your page. That takes JavaScript and cookies, and,
|
|
155
|
+
if your site sends a `Content-Security-Policy`, one that allows inline
|
|
156
|
+
scripts. The setting starts at **Serve them the normal page**, and older gates
|
|
157
|
+
ignore it.
|
|
158
|
+
|
|
143
159
|
## Behind a proxy or load balancer
|
|
144
160
|
|
|
145
161
|
The gate sends the network the address of an agent that claims to be a
|
|
@@ -172,7 +188,7 @@ offer `Cache-Control: no-store`, so a cache never serves an offer to a person.
|
|
|
172
188
|
## Check the install
|
|
173
189
|
|
|
174
190
|
To check the install, open `https://your-site.example/.well-known/constile`. The
|
|
175
|
-
gate answers with your site ID, `"gate_version": "constile-gate 0.
|
|
191
|
+
gate answers with your site ID, `"gate_version": "constile-gate 0.7.0"`, and the
|
|
176
192
|
platform. The install check on your site page reads the same file.
|
|
177
193
|
|
|
178
194
|
## How the gate behaves
|
data/lib/constile_gate/agents.rb
CHANGED
|
@@ -9,6 +9,9 @@ module ConstileGate
|
|
|
9
9
|
# The reference's /bot\b|crawler|spider|headless|scraper/, with an ASCII word boundary.
|
|
10
10
|
GENERIC_BOT = /bot(?![A-Za-z0-9_])|crawler|spider|headless|scraper/.freeze
|
|
11
11
|
QUOTED_HTTPS = %r{"(https://[^"]+)"}.freeze
|
|
12
|
+
# The cookie the offer page's fallback sets in a person's browser (any value). A request that carries it is never
|
|
13
|
+
# taken for a script posing as a browser.
|
|
14
|
+
BROWSER_COOKIE = "constile_browser".freeze
|
|
12
15
|
|
|
13
16
|
module_function
|
|
14
17
|
|
|
@@ -22,8 +25,18 @@ module ConstileGate
|
|
|
22
25
|
nil
|
|
23
26
|
end
|
|
24
27
|
|
|
25
|
-
#
|
|
26
|
-
|
|
28
|
+
# Whether a request with a browser's user agent leaves out what every browser sends: neither Accept-Language nor
|
|
29
|
+
# Sec-Fetch-Mode (a header counts when it's there at all, even empty), and no browser cookie.
|
|
30
|
+
def poses_as_browser?(headers)
|
|
31
|
+
return false if headers.has?("accept-language") || headers.has?("sec-fetch-mode")
|
|
32
|
+
|
|
33
|
+
(headers.get("cookie") || "").split(";", -1).none? { |part| JS.trim(part.split("=", -1)[0] || "") == BROWSER_COOKIE }
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Returns { kind: "agent" | "crawler" | "search" | "browser" | "unknown_client" | "disguised", agent: entry or nil }.
|
|
37
|
+
# "disguised" (a browser's user agent without the headers browsers send) comes back only when the caller asks for
|
|
38
|
+
# the check with disguised: true; a gate asks only when its site sends those an ad.
|
|
39
|
+
def classify_client(headers, registry, disguised: false)
|
|
27
40
|
ua = JS.downcase(headers.get("user-agent") || "")
|
|
28
41
|
# Longest token first, so "claude-searchbot" wins over a shorter token it contains. Ties keep registry order.
|
|
29
42
|
by_length = registry.each_with_index.sort_by { |a, i| [-JS.utf16_length(a["token"].to_s), i] }.map(&:first)
|
|
@@ -42,7 +55,9 @@ module ConstileGate
|
|
|
42
55
|
return { kind: "agent", agent: signer } if signer
|
|
43
56
|
return { kind: "unknown_client", agent: nil } if ua.empty?
|
|
44
57
|
# Never show an offer to a person: any browser-looking client without a bot marker is a browser.
|
|
45
|
-
|
|
58
|
+
if ua.start_with?("mozilla/") && !ua.match?(GENERIC_BOT)
|
|
59
|
+
return { kind: disguised && poses_as_browser?(headers) ? "disguised" : "browser", agent: nil }
|
|
60
|
+
end
|
|
46
61
|
|
|
47
62
|
{ kind: "unknown_client", agent: nil }
|
|
48
63
|
end
|
data/lib/constile_gate/gate.rb
CHANGED
|
@@ -287,20 +287,26 @@ module ConstileGate
|
|
|
287
287
|
cookies = cookie_names(request)
|
|
288
288
|
return deps.origin.call(request) if config["bypass_cookies"].any? { |c| cookies.key?(c) }
|
|
289
289
|
|
|
290
|
-
|
|
290
|
+
# A site that sends scripts posing as a browser an ad asks the classifier to look for them; otherwise they're browsers.
|
|
291
|
+
cls = Agents.classify_client(request.headers, config["agents"], disguised: config["disguised_clients"] == "offer")
|
|
292
|
+
disguised = false
|
|
291
293
|
if cls[:kind] == "agent"
|
|
292
294
|
return deps.origin.call(request) if cls[:agent]["tier"] == "blocked"
|
|
293
295
|
|
|
294
296
|
agent_id = cls[:agent]["id"]
|
|
295
297
|
elsif cls[:kind] == "unknown_client" && config["unknown_clients"] == "offer"
|
|
296
298
|
agent_id = nil
|
|
299
|
+
elsif cls[:kind] == "disguised"
|
|
300
|
+
# An unidentified client, whatever unknown_clients says. The network adds a way back to the page for a person.
|
|
301
|
+
agent_id = nil
|
|
302
|
+
disguised = true
|
|
297
303
|
else
|
|
298
304
|
return deps.origin.call(request)
|
|
299
305
|
end
|
|
300
306
|
|
|
301
307
|
if demand_only
|
|
302
308
|
# The site is in review: serve the page as usual, and tell the network an agent asked for it.
|
|
303
|
-
body = demand_request(request, url, agent_id, deps)
|
|
309
|
+
body = demand_request(request, url, agent_id, disguised, deps)
|
|
304
310
|
report = lambda do
|
|
305
311
|
call_network(deps, "/demand", method: "POST", body: body)
|
|
306
312
|
rescue StandardError => e
|
|
@@ -338,6 +344,7 @@ module ConstileGate
|
|
|
338
344
|
"user_agent" => request.headers.get("user-agent") || "",
|
|
339
345
|
"signals" => signals(request, deps, agent_id),
|
|
340
346
|
}
|
|
347
|
+
offer_req["disguised"] = true if disguised
|
|
341
348
|
# Web Bot Auth: forward the signature unchanged so the network verifies it, not the gate.
|
|
342
349
|
sig = signature(request, url)
|
|
343
350
|
offer_req["signature"] = sig if sig
|
|
@@ -522,9 +529,10 @@ module ConstileGate
|
|
|
522
529
|
{ "signature" => sig, "signature_input" => sig_input, "signature_agent" => request.headers.get("signature-agent"), "authority" => url.host }
|
|
523
530
|
end
|
|
524
531
|
|
|
525
|
-
def demand_request(request, url, agent_id, deps)
|
|
532
|
+
def demand_request(request, url, agent_id, disguised, deps)
|
|
526
533
|
out = { "url" => url.to_s, "agent_id" => agent_id, "user_agent" => request.headers.get("user-agent") || "",
|
|
527
534
|
"signals" => signals(request, deps, agent_id) }
|
|
535
|
+
out["disguised"] = true if disguised
|
|
528
536
|
sig = signature(request, url)
|
|
529
537
|
out["signature"] = sig if sig
|
|
530
538
|
out
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
module ConstileGate
|
|
2
|
-
VERSION = "0.
|
|
2
|
+
VERSION = "0.7.0".freeze
|
|
3
3
|
# Sent to the network with every call and shown in /.well-known/constile. The same in every port.
|
|
4
|
-
GATE_VERSION = "constile-gate 0.
|
|
4
|
+
GATE_VERSION = "constile-gate 0.7.0".freeze
|
|
5
5
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: constile_gate
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.7.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- CCC2C Inc.
|
|
8
|
-
autorequire:
|
|
9
8
|
bindir: bin
|
|
10
9
|
cert_chain: []
|
|
11
|
-
date: 2026-11-
|
|
10
|
+
date: 2026-11-20 00:00:00.000000000 Z
|
|
12
11
|
dependencies: []
|
|
13
12
|
description: Serves the Constile offer to AI agents that fetch a page for a person,
|
|
14
13
|
forwards the offer's callbacks to the Constile network, and passes everyone else
|
|
@@ -40,7 +39,6 @@ licenses:
|
|
|
40
39
|
metadata:
|
|
41
40
|
documentation_uri: https://constile.app/setup/rack
|
|
42
41
|
rubygems_mfa_required: 'true'
|
|
43
|
-
post_install_message:
|
|
44
42
|
rdoc_options: []
|
|
45
43
|
require_paths:
|
|
46
44
|
- lib
|
|
@@ -55,8 +53,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
55
53
|
- !ruby/object:Gem::Version
|
|
56
54
|
version: '0'
|
|
57
55
|
requirements: []
|
|
58
|
-
rubygems_version: 3.
|
|
59
|
-
signing_key:
|
|
56
|
+
rubygems_version: 3.6.7
|
|
60
57
|
specification_version: 4
|
|
61
58
|
summary: The Constile gate as Rack middleware, for Rails, Sinatra, and any Rack app.
|
|
62
59
|
test_files: []
|