nondisposable 0.4.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +59 -0
- data/README.md +185 -6
- data/app/models/nondisposable/disposable_domain.rb +53 -34
- data/app/models/nondisposable/source_snapshot.rb +66 -0
- data/data/known_typo_domains.txt +99 -0
- data/data/never_disposable.txt +480 -0
- data/data/privacy_relays.txt +42 -0
- data/data/public_suffix_list.dat +16571 -0
- data/gemfiles/rails_7.2.gemfile +1 -0
- data/gemfiles/rails_8.0.gemfile +1 -0
- data/gemfiles/rails_8.1.gemfile +1 -0
- data/lib/generators/nondisposable/install_generator.rb +1 -0
- data/lib/generators/nondisposable/templates/create_nondisposable_source_snapshots.rb.erb +42 -0
- data/lib/generators/nondisposable/templates/nondisposable.rb +29 -7
- data/lib/generators/nondisposable/upgrade_generator.rb +43 -0
- data/lib/nondisposable/confirmable_email.rb +75 -0
- data/lib/nondisposable/domain.rb +238 -0
- data/lib/nondisposable/domain_list_updater.rb +529 -88
- data/lib/nondisposable/email_validator.rb +51 -4
- data/lib/nondisposable/known_typos.rb +64 -0
- data/lib/nondisposable/precedence.rb +209 -0
- data/lib/nondisposable/public_suffix.rb +147 -0
- data/lib/nondisposable/public_suffix_list_updater.rb +88 -0
- data/lib/nondisposable/source.rb +233 -0
- data/lib/nondisposable/source_fetcher.rb +69 -0
- data/lib/nondisposable/source_merge.rb +127 -0
- data/lib/nondisposable/suggestion.rb +201 -31
- data/lib/nondisposable/tld.rb +8 -3
- data/lib/nondisposable/version.rb +1 -1
- data/lib/nondisposable.rb +68 -13
- data/lib/tasks/nondisposable.rake +29 -0
- metadata +18 -2
|
@@ -20,15 +20,44 @@ module ActiveModel
|
|
|
20
20
|
# verdict with the fix attached. So whenever we can name a correction we
|
|
21
21
|
# use that message, whether the TLD was invalid or merely lookalike. The
|
|
22
22
|
# rule is: a suggestion, if we have one, always phrases the rejection.
|
|
23
|
+
#
|
|
24
|
+
# HARD vs CONFIRMABLE — WHICH LOOKALIKES CAN BE WAVED THROUGH
|
|
25
|
+
#
|
|
26
|
+
# A lookalike error is only as strong as the evidence behind it:
|
|
27
|
+
#
|
|
28
|
+
# - Invalid TLD (`gmail.con`, `gmailmcom`): HARD. No mail is delivered to
|
|
29
|
+
# a TLD that does not exist, so there is nothing to confirm.
|
|
30
|
+
# - Reviewed typo (data/known_typo_domains.txt, `gmial.com`): HARD. Someone
|
|
31
|
+
# checked; nobody has a mailbox there.
|
|
32
|
+
# - Any other lookalike (`gmail.co`, but also `ge.com` → me.com): a GUESS,
|
|
33
|
+
# so CONFIRMABLE. The error is added unless the record says the user
|
|
34
|
+
# confirmed the address as typed (Nondisposable::ConfirmableEmail). A
|
|
35
|
+
# real `ge.com` employee resubmits once and is never locked out.
|
|
36
|
+
#
|
|
37
|
+
# `true` and `:confirmable` mean the same thing. Before 0.5.0 `true` refused
|
|
38
|
+
# every lookalike outright, which measured against the Tranco top 1M would
|
|
39
|
+
# have refused staff at General Electric, EA and Aon (see CHANGELOG).
|
|
23
40
|
def validate_each(record, attribute, value)
|
|
24
41
|
return if value.blank?
|
|
25
42
|
|
|
26
43
|
begin
|
|
27
|
-
domain
|
|
28
|
-
|
|
44
|
+
# Without an "@" the whole value is judged as a domain, as it always
|
|
45
|
+
# has been — someone who typed only `tempmail.com` is still refused.
|
|
46
|
+
# Unicode form: DisposableDomain canonicalises it, and needs the
|
|
47
|
+
# unencoded spelling to match rows stored before 0.5.0.
|
|
48
|
+
domain = value.to_s.include?('@') ? Nondisposable::Domain.from_email(value, unicode: true) : Nondisposable::Domain.unicode(value)
|
|
49
|
+
return if domain.empty? # Invalid email format
|
|
29
50
|
|
|
30
51
|
config = Nondisposable.configuration
|
|
31
52
|
|
|
53
|
+
# A domain longer than DNS allows (RFC 1035; RFC 5321 caps it at
|
|
54
|
+
# 255) cannot receive mail and is in no list, so "not disposable"
|
|
55
|
+
# would be a lie: refuse it as invalid, whatever else is switched
|
|
56
|
+
# on. Only for real addresses — a value without "@" is `format:`'s.
|
|
57
|
+
if value.to_s.include?('@') && Nondisposable::Domain.too_long?(domain)
|
|
58
|
+
return reject(record, attribute, config.invalid_tld_error_message)
|
|
59
|
+
end
|
|
60
|
+
|
|
32
61
|
if Nondisposable::DisposableDomain.disposable?(domain)
|
|
33
62
|
return reject(record, attribute, options[:message] || config.error_message)
|
|
34
63
|
end
|
|
@@ -45,13 +74,31 @@ module ActiveModel
|
|
|
45
74
|
# production-shaped typo that missed the dot), which fails "must end in a real TLD"
|
|
46
75
|
# just as surely as `.con` does.
|
|
47
76
|
tld = Nondisposable::Tld.extract(value)
|
|
48
|
-
|
|
77
|
+
impossible = tld.nil? || !Nondisposable::Tld.valid?(tld)
|
|
78
|
+
invalid = config.check_tld && impossible
|
|
49
79
|
blocked = config.check_tld && !invalid && Nondisposable::Tld.blocked?(tld)
|
|
50
80
|
|
|
51
81
|
if invalid || config.reject_lookalike_domains
|
|
52
82
|
suggestion = Nondisposable::Suggestion.for(value)
|
|
83
|
+
# `@gmial.com` has no local part to build "Did you mean ..." from,
|
|
84
|
+
# but it is still a reviewed typo: name the domain instead.
|
|
85
|
+
if suggestion.nil? && config.lookalike_distance.to_i.positive?
|
|
86
|
+
suggestion = Nondisposable::KnownTypos.correction_for(value)
|
|
87
|
+
end
|
|
53
88
|
if suggestion
|
|
54
|
-
|
|
89
|
+
message = config.lookalike_error_message.sub('%{suggestion}', suggestion)
|
|
90
|
+
# Hard: no confirmation can make an impossible TLD deliverable —
|
|
91
|
+
# whether or not the host turned check_tld on, `gmail.con` cannot
|
|
92
|
+
# receive the confirmation email — and a reviewed typo is not a
|
|
93
|
+
# guess. The error stays a plain message, exactly as before 0.5.0.
|
|
94
|
+
return reject(record, attribute, message) if impossible || Nondisposable::KnownTypos.include?(value)
|
|
95
|
+
|
|
96
|
+
unless Nondisposable::ConfirmableEmail.confirmed?(record, attribute)
|
|
97
|
+
# Typed, so a form can tell this error apart and offer the
|
|
98
|
+
# "yes, this is right" checkbox: errors.of_kind?(:email, :lookalike_domain).
|
|
99
|
+
record.errors.add(attribute, :lookalike_domain, message: message, suggestion: suggestion)
|
|
100
|
+
return
|
|
101
|
+
end
|
|
55
102
|
end
|
|
56
103
|
end
|
|
57
104
|
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Nondisposable
|
|
4
|
+
# The only lookalike domains this gem refuses outright.
|
|
5
|
+
#
|
|
6
|
+
# "One edit from gmail.com" is a guess, and offline a guess cannot tell
|
|
7
|
+
# `gmial.com` (a typo) from `ge.com` (General Electric, one edit from me.com).
|
|
8
|
+
# So a guess may only ever produce a CONFIRMABLE error — the user can say
|
|
9
|
+
# "yes, that is my address" and carry on. Refusing without that way out needs
|
|
10
|
+
# something stronger than a guess: a domain somebody looked at and found to be
|
|
11
|
+
# nothing but a misspelling of a provider — parked or for sale under the
|
|
12
|
+
# provider's misspelled name, redirecting to the provider, or squatted by a
|
|
13
|
+
# known typosquatter.
|
|
14
|
+
#
|
|
15
|
+
# Those reviewed domains live in data/known_typo_domains.txt, each with the
|
|
16
|
+
# correction it is a typo OF. The evidence behind every entry is in
|
|
17
|
+
# test/fixtures/lookalike_corpus.csv, and the test suite fails if an entry
|
|
18
|
+
# loses it. Nothing generates this list automatically; see the file's header
|
|
19
|
+
# for the bar an entry has to clear.
|
|
20
|
+
module KnownTypos
|
|
21
|
+
LIST_PATH = File.expand_path('../../data/known_typo_domains.txt', __dir__)
|
|
22
|
+
|
|
23
|
+
class << self
|
|
24
|
+
# Is this domain (or address) a reviewed typo?
|
|
25
|
+
def include?(email_or_domain)
|
|
26
|
+
!correction_for(email_or_domain).nil?
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# The provider domain it is a typo of, or nil.
|
|
30
|
+
def correction_for(email_or_domain)
|
|
31
|
+
value = email_or_domain.to_s
|
|
32
|
+
domain = value.include?('@') ? Domain.from_email(value) : Domain.normalize(value)
|
|
33
|
+
return nil if domain.empty?
|
|
34
|
+
# A host who added a provider has told us its users are real, which
|
|
35
|
+
# outranks any review we did without knowing about them.
|
|
36
|
+
return nil if host_providers.include?(domain)
|
|
37
|
+
|
|
38
|
+
all[domain]
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# typo => correction, frozen. Read once per process.
|
|
42
|
+
def all
|
|
43
|
+
@all ||= File.readlines(LIST_PATH, chomp: true).each_with_object({}) do |line, map|
|
|
44
|
+
value = line.sub(/#.*/, '').strip
|
|
45
|
+
next if value.empty?
|
|
46
|
+
|
|
47
|
+
typo, correction = value.split(/\s+/, 2)
|
|
48
|
+
map[Domain.normalize(typo)] = Domain.normalize(correction)
|
|
49
|
+
end.freeze
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def reload!
|
|
53
|
+
@all = nil
|
|
54
|
+
self
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
private
|
|
58
|
+
|
|
59
|
+
def host_providers
|
|
60
|
+
Array(Nondisposable.configuration.additional_email_providers).map { |d| Domain.normalize(d) }
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'set'
|
|
4
|
+
|
|
5
|
+
module Nondisposable
|
|
6
|
+
# Who wins when two lists disagree about a domain. ONE table, used both when
|
|
7
|
+
# an address is checked (#decide) and when the stored list is built
|
|
8
|
+
# (#filter), so the table and the check can never tell different stories:
|
|
9
|
+
#
|
|
10
|
+
# 1. config.excluded_domains (host) → allowed
|
|
11
|
+
# 2. data/never_disposable.txt (gem) → allowed real providers and ISPs
|
|
12
|
+
# 3. config.additional_domains (host) → blocked
|
|
13
|
+
# 4. data/privacy_relays.txt (gem) → config.block_privacy_relays decides
|
|
14
|
+
# 5. the merged upstream sources → blocked
|
|
15
|
+
#
|
|
16
|
+
# Before 0.6.0, additional_domains was checked FIRST, so a host could not
|
|
17
|
+
# exclude a domain it had also added. Now the host's exclusion wins: an
|
|
18
|
+
# allowlist entry is the more deliberate statement, and "a real person
|
|
19
|
+
# cannot sign up" is the costlier mistake.
|
|
20
|
+
#
|
|
21
|
+
# Row 2 outranking row 3 is the deliberate trade-off of this table: a host
|
|
22
|
+
# can no longer block a real provider (say `mail.ru`) through
|
|
23
|
+
# additional_domains. That list is for throwaway domains the upstream lists
|
|
24
|
+
# missed; refusing a whole real provider is a business rule, and belongs in
|
|
25
|
+
# the host's own validation where it can say so to the user.
|
|
26
|
+
#
|
|
27
|
+
# HOW EACH ROW MATCHES
|
|
28
|
+
#
|
|
29
|
+
# - excluded_domains: an exact match allows; an excluded parent can never be
|
|
30
|
+
# the REASON for a block, but a more specific listed subdomain still blocks
|
|
31
|
+
# (unchanged since 0.3.0).
|
|
32
|
+
# - never_disposable and privacy relays protect their whole namespace
|
|
33
|
+
# (`bak.rr.com` is covered by `rr.com`), whatever check_parent_domains says:
|
|
34
|
+
# upstream lists really do carry ISP subdomains like that one.
|
|
35
|
+
# - additional_domains and the stored list follow check_parent_domains.
|
|
36
|
+
# - No public suffix ever matches anything (Nondisposable::PublicSuffix).
|
|
37
|
+
module Precedence
|
|
38
|
+
NEVER_DISPOSABLE_PATH = File.expand_path('../../data/never_disposable.txt', __dir__)
|
|
39
|
+
PRIVACY_RELAYS_PATH = File.expand_path('../../data/privacy_relays.txt', __dir__)
|
|
40
|
+
EMAIL_PROVIDERS_PATH = File.expand_path('../../data/email_providers.txt', __dir__)
|
|
41
|
+
|
|
42
|
+
# A syntactically possible DNS name, after Domain.normalize: at least two
|
|
43
|
+
# labels of 1-63 characters, 253 in total, and a final label that is not
|
|
44
|
+
# all digits (that would be an IP address, not a domain). Underscores are
|
|
45
|
+
# tolerated inside labels — they are legal in DNS, if not in hostnames, and
|
|
46
|
+
# rejecting them would only ever make the list shorter, never safer.
|
|
47
|
+
LABEL = /\A[a-z0-9_](?:[a-z0-9_-]{0,61}[a-z0-9_])?\z/
|
|
48
|
+
|
|
49
|
+
class << self
|
|
50
|
+
# ---- Checking one address ---------------------------------------------
|
|
51
|
+
|
|
52
|
+
# The verdict for a normalized domain, given the stored-list candidates
|
|
53
|
+
# DisposableDomain.match_candidates produced for it:
|
|
54
|
+
#
|
|
55
|
+
# [:allow, nil] rows 1, 2 or 4 said no
|
|
56
|
+
# [:block, nil] rows 3 or 4 said yes
|
|
57
|
+
# [:lookup, candidates] ask the stored list (row 5) about these
|
|
58
|
+
def decide(domain, candidates)
|
|
59
|
+
config = Nondisposable.configuration
|
|
60
|
+
excluded = configured_set(config.excluded_domains)
|
|
61
|
+
|
|
62
|
+
return [:allow, nil] if excluded.include?(domain)
|
|
63
|
+
return [:allow, nil] if namespace_hit(domain, never_disposable)
|
|
64
|
+
|
|
65
|
+
remaining = candidates.reject { |candidate| excluded.include?(candidate) }
|
|
66
|
+
|
|
67
|
+
additional = configured_set(config.additional_domains)
|
|
68
|
+
return [:block, nil] if remaining.any? { |candidate| additional.include?(candidate) }
|
|
69
|
+
|
|
70
|
+
# The relay ALLOWANCE covers the whole relay namespace, whatever is
|
|
71
|
+
# excluded: adding an exclusion must never turn an allowed address into
|
|
72
|
+
# a refused one (filter drops every relay-namespace row the same way).
|
|
73
|
+
# Under block_privacy_relays an excluded relay name stops being a
|
|
74
|
+
# reason to block, like any other exclusion.
|
|
75
|
+
relay_names = namespace(domain).select { |name| privacy_relays.include?(name) }
|
|
76
|
+
if relay_names.any?
|
|
77
|
+
return [:allow, nil] unless config.block_privacy_relays
|
|
78
|
+
return [:block, nil] if relay_names.any? { |name| !excluded.include?(name) }
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
[:lookup, remaining]
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# ---- Building the stored list -----------------------------------------
|
|
85
|
+
|
|
86
|
+
Result = Struct.new(:kept, :dropped) do
|
|
87
|
+
# { reason => [domains] }, for logging.
|
|
88
|
+
def dropped_by_reason
|
|
89
|
+
dropped.group_by { |_domain, reason| reason }.transform_values { |pairs| pairs.map(&:first).sort }
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Applies the table to a list about to be stored: every upstream domain
|
|
94
|
+
# is first put in Domain.normalize's canonical form, so every write path
|
|
95
|
+
# (fetch, seed, bundled fallback) stores exactly the name a lookup asks
|
|
96
|
+
# for, whoever the caller is. Each then goes through rows 1, 2 and 4 and
|
|
97
|
+
# the validity checks; the host's additional_domains are added through
|
|
98
|
+
# row 2 and the validity checks.
|
|
99
|
+
#
|
|
100
|
+
# Exclusions are exact here, as they always were: the stored list is not
|
|
101
|
+
# where parent-level exclusions take effect, #decide is.
|
|
102
|
+
#
|
|
103
|
+
# host_policy: false applies only the gem's own rules (validity, public
|
|
104
|
+
# suffixes, never-disposable, and relays always dropped), ignoring every
|
|
105
|
+
# host setting. The updater's size guards count that, so their numbers
|
|
106
|
+
# never depend on the host's configuration.
|
|
107
|
+
def filter(domains, host_policy: true)
|
|
108
|
+
config = Nondisposable.configuration
|
|
109
|
+
excluded = host_policy ? configured_set(config.excluded_domains) : Set.new
|
|
110
|
+
block_relays = host_policy && config.block_privacy_relays
|
|
111
|
+
kept = Set.new
|
|
112
|
+
dropped = []
|
|
113
|
+
|
|
114
|
+
domains.each do |raw|
|
|
115
|
+
domain = Domain.normalize(raw)
|
|
116
|
+
next if domain.empty?
|
|
117
|
+
|
|
118
|
+
reason = invalid_reason(domain) ||
|
|
119
|
+
(:excluded if excluded.include?(domain)) ||
|
|
120
|
+
(:never_disposable if namespace_hit(domain, never_disposable)) ||
|
|
121
|
+
(:privacy_relay if !block_relays && namespace_hit(domain, privacy_relays))
|
|
122
|
+
reason ? dropped << [domain, reason] : kept << domain
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
Array(host_policy ? config.additional_domains : []).each do |raw|
|
|
126
|
+
domain = Domain.normalize(raw)
|
|
127
|
+
next if domain.empty? || excluded.include?(domain)
|
|
128
|
+
|
|
129
|
+
reason = (invalid_reason(domain) && :invalid_additional_domain) ||
|
|
130
|
+
(:additional_domain_is_never_disposable if namespace_hit(domain, never_disposable))
|
|
131
|
+
reason ? dropped << [domain, reason] : kept << domain
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
Result.new(kept, dropped)
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# nil when `domain` (normalized) may be stored, otherwise why not.
|
|
138
|
+
def invalid_reason(domain)
|
|
139
|
+
return :invalid unless hostname?(domain)
|
|
140
|
+
return :public_suffix if PublicSuffix.public_suffix?(domain)
|
|
141
|
+
|
|
142
|
+
nil
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
def hostname?(domain)
|
|
146
|
+
return false if domain.empty? || domain.length > 253
|
|
147
|
+
|
|
148
|
+
labels = domain.split('.', -1)
|
|
149
|
+
labels.size >= 2 &&
|
|
150
|
+
labels.all? { |label| label.length <= 63 && LABEL.match?(label) } &&
|
|
151
|
+
!labels.last.match?(/\A\d+\z/)
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
# ---- The bundled lists -------------------------------------------------
|
|
155
|
+
|
|
156
|
+
# data/never_disposable.txt + data/email_providers.txt, minus the relays.
|
|
157
|
+
# Unioned at runtime so a provider added for the typo checks is protected
|
|
158
|
+
# here too, without anybody remembering to copy it.
|
|
159
|
+
def never_disposable
|
|
160
|
+
@never_disposable ||= (read_list(NEVER_DISPOSABLE_PATH) | read_list(EMAIL_PROVIDERS_PATH)).subtract(privacy_relays).freeze
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
def privacy_relays
|
|
164
|
+
@privacy_relays ||= read_list(PRIVACY_RELAYS_PATH).freeze
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
def reload!
|
|
168
|
+
@never_disposable = nil
|
|
169
|
+
@privacy_relays = nil
|
|
170
|
+
self
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
# The domain and each of its parents down to its registrable domain:
|
|
174
|
+
# `a.b.rr.com` => ["a.b.rr.com", "b.rr.com", "rr.com"]. Empty for a public
|
|
175
|
+
# suffix. Never climbs past the registrable domain, so `sch.uk` is not in
|
|
176
|
+
# the namespace of `school.county.sch.uk`.
|
|
177
|
+
def namespace(domain)
|
|
178
|
+
registrable = PublicSuffix.registrable_domain(domain)
|
|
179
|
+
return [] unless registrable
|
|
180
|
+
|
|
181
|
+
labels = domain.split('.')
|
|
182
|
+
(0..(labels.size - registrable.count('.') - 1)).map { |i| labels.drop(i).join('.') }
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
# The most specific name in `domain`'s namespace that `set` contains.
|
|
186
|
+
def namespace_hit(domain, set)
|
|
187
|
+
namespace(domain).find { |name| set.include?(name) }
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
private
|
|
191
|
+
|
|
192
|
+
# Host-supplied, so normalized on every read rather than trusted.
|
|
193
|
+
def configured_set(values)
|
|
194
|
+
Array(values).each_with_object(Set.new) do |value, set|
|
|
195
|
+
domain = Domain.normalize(value)
|
|
196
|
+
set << domain unless domain.empty?
|
|
197
|
+
end
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
# One domain per line; `#` starts a comment, inline or whole-line.
|
|
201
|
+
def read_list(path)
|
|
202
|
+
File.foreach(path, encoding: 'UTF-8').each_with_object(Set.new) do |line, set|
|
|
203
|
+
domain = Domain.normalize(line.sub(/#.*/, ''))
|
|
204
|
+
set << domain unless domain.empty?
|
|
205
|
+
end
|
|
206
|
+
end
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
end
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'set'
|
|
4
|
+
|
|
5
|
+
module Nondisposable
|
|
6
|
+
# Is this domain a PUBLIC SUFFIX — a name under which anybody can register
|
|
7
|
+
# their own domain, like `com`, `co.uk`, `com.br` or `github.io`?
|
|
8
|
+
#
|
|
9
|
+
# WHY THE BLOCKLIST NEEDS TO KNOW
|
|
10
|
+
#
|
|
11
|
+
# Parent-domain matching blocks `x.tempmail.com` because `tempmail.com` is on
|
|
12
|
+
# the list. That is exactly right for a registrable domain and exactly wrong
|
|
13
|
+
# for a public suffix: one bad line reading `com.br` in any upstream list
|
|
14
|
+
# would refuse every Brazilian company's staff, and nothing in a domain count
|
|
15
|
+
# or a file size would look unusual. So no public suffix is ever stored, ever
|
|
16
|
+
# accepted from `additional_domains`, and ever used as a parent-match
|
|
17
|
+
# candidate. That last one matters on its own: it keeps a row written by an
|
|
18
|
+
# older version of this gem from doing any damage either.
|
|
19
|
+
#
|
|
20
|
+
# WHY A BUNDLED SNAPSHOT AND NOT THE `public_suffix` GEM
|
|
21
|
+
#
|
|
22
|
+
# The same reason as Nondisposable::Tld: we need one yes/no answer, offline,
|
|
23
|
+
# on every signup, with no network and no extra runtime dependency. The list
|
|
24
|
+
# is the canonical file from https://publicsuffix.org/list/public_suffix_list.dat
|
|
25
|
+
# (MPL 2.0, redistributed unmodified), ~10,000 rules in ~330 KB, parsed once
|
|
26
|
+
# into frozen Sets. Both sections are honoured: the ICANN section (`co.uk`)
|
|
27
|
+
# and the private one (`github.io`, `blogspot.com`), because in both cases the
|
|
28
|
+
# names below belong to unrelated people. Refresh it with
|
|
29
|
+
# `rake nondisposable:psl:update` when cutting a release.
|
|
30
|
+
#
|
|
31
|
+
# STALENESS
|
|
32
|
+
#
|
|
33
|
+
# A suffix added after the snapshot is simply unknown: we would accept a
|
|
34
|
+
# blocklist entry for it, which is today's behaviour, so a stale snapshot can
|
|
35
|
+
# only ever be less protective, never lock anybody out.
|
|
36
|
+
module PublicSuffix
|
|
37
|
+
LIST_PATH = File.expand_path('../../data/public_suffix_list.dat', __dir__)
|
|
38
|
+
|
|
39
|
+
class << self
|
|
40
|
+
# True when `domain` is itself a public suffix. Single-label names (bare
|
|
41
|
+
# TLDs, known or not) always are: that is the PSL's implicit `*` rule.
|
|
42
|
+
# Expects a normalized domain (see Nondisposable::Domain.normalize).
|
|
43
|
+
def public_suffix?(domain)
|
|
44
|
+
domain = domain.to_s
|
|
45
|
+
return false if domain.empty?
|
|
46
|
+
|
|
47
|
+
labels = domain.split('.')
|
|
48
|
+
return true if labels.size == 1
|
|
49
|
+
|
|
50
|
+
list = rules
|
|
51
|
+
return false if list[:exceptions].include?(domain)
|
|
52
|
+
return true if list[:exact].include?(domain)
|
|
53
|
+
|
|
54
|
+
list[:wildcards].include?(labels.drop(1).join('.'))
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# The registrable domain `domain` belongs to: its longest public suffix
|
|
58
|
+
# plus one label. `school.county.sch.uk` => itself (`county.sch.uk` is a
|
|
59
|
+
# suffix via `*.sch.uk`); `x.tempmail.com` => `tempmail.com`. nil when
|
|
60
|
+
# `domain` is itself a public suffix.
|
|
61
|
+
#
|
|
62
|
+
# Everything ABOVE this name belongs to the registry or to unrelated
|
|
63
|
+
# registrants, so no list entry up there may ever match `domain` — not
|
|
64
|
+
# even a name like `sch.uk` that is not itself a suffix but sits above a
|
|
65
|
+
# wildcard one.
|
|
66
|
+
def registrable_domain(domain)
|
|
67
|
+
labels = domain.to_s.split('.')
|
|
68
|
+
return nil if labels.empty? || public_suffix?(domain.to_s)
|
|
69
|
+
|
|
70
|
+
# An exception rule prevails (`!city.kawasaki.jp` makes its parent the
|
|
71
|
+
# suffix); otherwise the longest suffix that is public wins.
|
|
72
|
+
exceptions = rules[:exceptions]
|
|
73
|
+
exception = (1..labels.size).find { |n| exceptions.include?(labels.last(n).join('.')) }
|
|
74
|
+
suffix_size = if exception
|
|
75
|
+
exception - 1
|
|
76
|
+
else
|
|
77
|
+
(1...labels.size).select { |n| public_suffix?(labels.last(n).join('.')) }.max || 1
|
|
78
|
+
end
|
|
79
|
+
labels.last(suffix_size + 1).join('.')
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# IANA-style provenance for health checks: "2026-10-16_23-32-13_UTC".
|
|
83
|
+
def version
|
|
84
|
+
@version ||= begin
|
|
85
|
+
line = File.foreach(LIST_PATH).first(20).find { |l| l.start_with?('// VERSION:') }
|
|
86
|
+
line&.delete_prefix('// VERSION:')&.strip
|
|
87
|
+
rescue StandardError
|
|
88
|
+
nil
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def rule_count
|
|
93
|
+
list = rules
|
|
94
|
+
list[:exact].size + list[:wildcards].size + list[:exceptions].size
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# Reset the memoized rules. Called by the updater after it rewrites the
|
|
98
|
+
# file, and by tests.
|
|
99
|
+
def reload!
|
|
100
|
+
@rules = nil
|
|
101
|
+
@version = nil
|
|
102
|
+
self
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# Parses PSL text into the three rule kinds. Public so the updater can
|
|
106
|
+
# judge a downloaded body before it replaces the bundled one.
|
|
107
|
+
#
|
|
108
|
+
# Rules are stored in their ASCII (punycode) form, because that is the
|
|
109
|
+
# form Domain.normalize produces. A unicode rule we cannot convert is
|
|
110
|
+
# kept as-is: it then simply never matches, which errs towards today's
|
|
111
|
+
# behaviour rather than towards refusing anybody.
|
|
112
|
+
def parse(text)
|
|
113
|
+
exact = Set.new
|
|
114
|
+
wildcards = Set.new
|
|
115
|
+
exceptions = Set.new
|
|
116
|
+
|
|
117
|
+
text.each_line do |line|
|
|
118
|
+
rule = line.strip
|
|
119
|
+
next if rule.empty? || rule.start_with?('//')
|
|
120
|
+
|
|
121
|
+
rule = rule.split(/\s/, 2).first # the PSL ends a rule at whitespace
|
|
122
|
+
if rule.start_with?('!')
|
|
123
|
+
exceptions << ascii(rule.delete_prefix('!'))
|
|
124
|
+
elsif rule.start_with?('*.')
|
|
125
|
+
wildcards << ascii(rule.delete_prefix('*.'))
|
|
126
|
+
else
|
|
127
|
+
exact << ascii(rule)
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
{ exact: exact.freeze, wildcards: wildcards.freeze, exceptions: exceptions.freeze }.freeze
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
private
|
|
135
|
+
|
|
136
|
+
def rules
|
|
137
|
+
@rules ||= parse(File.read(LIST_PATH, encoding: 'UTF-8'))
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def ascii(rule)
|
|
141
|
+
Nondisposable::Domain.normalize(rule)
|
|
142
|
+
rescue StandardError
|
|
143
|
+
rule.downcase
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
end
|
|
147
|
+
end
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'net/http'
|
|
4
|
+
require 'uri'
|
|
5
|
+
|
|
6
|
+
module Nondisposable
|
|
7
|
+
# Refreshes the vendored Public Suffix List snapshot.
|
|
8
|
+
#
|
|
9
|
+
# ⚠️ A MAINTENANCE task, like TldListUpdater: it rewrites a file inside the
|
|
10
|
+
# gem, so run it in a checkout before cutting a release:
|
|
11
|
+
#
|
|
12
|
+
# rake nondisposable:psl:update
|
|
13
|
+
#
|
|
14
|
+
# Host apps never need it. A stale PSL can only be LESS protective (a suffix
|
|
15
|
+
# created after the snapshot is not known to be one), never lock anyone out.
|
|
16
|
+
class PublicSuffixListUpdater
|
|
17
|
+
# The PSL maintainers ask to be fetched from here and only here.
|
|
18
|
+
LIST_URL = 'https://publicsuffix.org/list/public_suffix_list.dat'
|
|
19
|
+
|
|
20
|
+
OPEN_TIMEOUT = 10
|
|
21
|
+
READ_TIMEOUT = 10
|
|
22
|
+
|
|
23
|
+
# The list holds ~10,000 rules. A body with a fraction of that is an error
|
|
24
|
+
# page or a truncated download, not the internet shrinking.
|
|
25
|
+
MINIMUM_PLAUSIBLE_RULE_COUNT = 5_000
|
|
26
|
+
|
|
27
|
+
class << self
|
|
28
|
+
# Returns the number of rules written, or nil if nothing was written.
|
|
29
|
+
def update(path: Nondisposable::PublicSuffix::LIST_PATH)
|
|
30
|
+
log :info, "Fetching the Public Suffix List from #{LIST_URL}..."
|
|
31
|
+
|
|
32
|
+
response = fetch
|
|
33
|
+
unless response.is_a?(Net::HTTPSuccess)
|
|
34
|
+
log :error, "Failed to download the Public Suffix List. HTTP status: #{response.code}. Keeping the existing snapshot."
|
|
35
|
+
return nil
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
body = response.body.to_s.dup.force_encoding(Encoding::UTF_8)
|
|
39
|
+
rules = Nondisposable::PublicSuffix.parse(body)
|
|
40
|
+
count = rules.values.sum(&:size)
|
|
41
|
+
|
|
42
|
+
unless body.include?('===BEGIN ICANN DOMAINS===') && count >= MINIMUM_PLAUSIBLE_RULE_COUNT
|
|
43
|
+
log :error, "Refusing to write a Public Suffix List with only #{count} rules (expected at least #{MINIMUM_PLAUSIBLE_RULE_COUNT} and an ICANN section). Keeping the existing snapshot."
|
|
44
|
+
return nil
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Written whole, then moved into place, so an interrupted run can never
|
|
48
|
+
# leave half a list behind.
|
|
49
|
+
tmp = "#{path}.tmp"
|
|
50
|
+
File.write(tmp, body)
|
|
51
|
+
File.rename(tmp, path)
|
|
52
|
+
Nondisposable::PublicSuffix.reload!
|
|
53
|
+
|
|
54
|
+
log :info, "Wrote #{count} rules to #{path} (#{Nondisposable::PublicSuffix.version})."
|
|
55
|
+
count
|
|
56
|
+
rescue StandardError => e
|
|
57
|
+
log :error, "Could not update the Public Suffix List: #{e.class}: #{e.message}. Keeping the existing snapshot."
|
|
58
|
+
nil
|
|
59
|
+
ensure
|
|
60
|
+
File.delete(tmp) if tmp && File.exist?(tmp)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
private
|
|
64
|
+
|
|
65
|
+
def fetch
|
|
66
|
+
uri = URI(LIST_URL)
|
|
67
|
+
Net::HTTP.start(
|
|
68
|
+
uri.host,
|
|
69
|
+
uri.port,
|
|
70
|
+
use_ssl: uri.scheme == 'https',
|
|
71
|
+
open_timeout: OPEN_TIMEOUT,
|
|
72
|
+
read_timeout: READ_TIMEOUT
|
|
73
|
+
) { |http| http.get(uri.request_uri) }
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# Usable from a bare `rake` run in a gem checkout, where there is no Rails
|
|
77
|
+
# logger to talk to.
|
|
78
|
+
def log(level, message)
|
|
79
|
+
message = "[nondisposable] #{message}"
|
|
80
|
+
if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
|
|
81
|
+
Rails.logger.public_send(level, message)
|
|
82
|
+
else
|
|
83
|
+
warn message
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
end
|