nondisposable 0.3.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +73 -0
  3. data/README.md +305 -3
  4. data/Rakefile +5 -0
  5. data/app/models/nondisposable/disposable_domain.rb +53 -34
  6. data/app/models/nondisposable/source_snapshot.rb +66 -0
  7. data/data/email_providers.txt +276 -0
  8. data/data/iana_tlds.txt +1439 -0
  9. data/data/known_typo_domains.txt +99 -0
  10. data/data/never_disposable.txt +480 -0
  11. data/data/privacy_relays.txt +42 -0
  12. data/data/public_suffix_list.dat +16571 -0
  13. data/gemfiles/rails_7.2.gemfile +1 -0
  14. data/gemfiles/rails_8.0.gemfile +1 -0
  15. data/gemfiles/rails_8.1.gemfile +1 -0
  16. data/lib/generators/nondisposable/install_generator.rb +1 -0
  17. data/lib/generators/nondisposable/templates/create_nondisposable_source_snapshots.rb.erb +42 -0
  18. data/lib/generators/nondisposable/templates/nondisposable.rb +67 -1
  19. data/lib/generators/nondisposable/upgrade_generator.rb +43 -0
  20. data/lib/nondisposable/confirmable_email.rb +75 -0
  21. data/lib/nondisposable/domain.rb +238 -0
  22. data/lib/nondisposable/domain_list_updater.rb +510 -88
  23. data/lib/nondisposable/email_validator.rb +101 -3
  24. data/lib/nondisposable/known_typos.rb +64 -0
  25. data/lib/nondisposable/precedence.rb +209 -0
  26. data/lib/nondisposable/public_suffix.rb +147 -0
  27. data/lib/nondisposable/public_suffix_list_updater.rb +88 -0
  28. data/lib/nondisposable/source.rb +233 -0
  29. data/lib/nondisposable/source_fetcher.rb +69 -0
  30. data/lib/nondisposable/source_merge.rb +127 -0
  31. data/lib/nondisposable/suggestion.rb +336 -0
  32. data/lib/nondisposable/tld.rb +190 -0
  33. data/lib/nondisposable/tld_list_updater.rb +101 -0
  34. data/lib/nondisposable/version.rb +1 -1
  35. data/lib/nondisposable.rb +153 -6
  36. data/lib/tasks/nondisposable.rake +66 -0
  37. metadata +24 -2
@@ -3,16 +3,107 @@
3
3
  module ActiveModel
4
4
  module Validations
5
5
  class NondisposableValidator < EachValidator
6
+ # Three independent questions about one address, in the order a human
7
+ # would ask them, and AT MOST ONE error however many of them fire —
8
+ # stacking "provider is not allowed" on top of "did you mean gmail.com?"
9
+ # helps nobody:
10
+ #
11
+ # 1. Is this a throwaway provider? (always on — the gem's whole job)
12
+ # 2. Can we name the address they meant? (config.reject_lookalike_domains,
13
+ # or any invalid TLD we can correct — see below)
14
+ # 3. Is the TLD real, and allowed? (config.check_tld)
15
+ #
16
+ # WHY A NAMEABLE CORRECTION OUTRANKS THE GENERIC TLD ERROR
17
+ #
18
+ # `user@gmail.con` fails both 2 and 3. "Doesn't look like a real email
19
+ # address" is true but useless; "Did you mean user@gmail.com?" is the same
20
+ # verdict with the fix attached. So whenever we can name a correction we
21
+ # use that message, whether the TLD was invalid or merely lookalike. The
22
+ # rule is: a suggestion, if we have one, always phrases the rejection.
23
+ #
24
+ # HARD vs CONFIRMABLE — WHICH LOOKALIKES CAN BE WAVED THROUGH
25
+ #
26
+ # A lookalike error is only as strong as the evidence behind it:
27
+ #
28
+ # - Invalid TLD (`gmail.con`, `gmailmcom`): HARD. No mail is delivered to
29
+ # a TLD that does not exist, so there is nothing to confirm.
30
+ # - Reviewed typo (data/known_typo_domains.txt, `gmial.com`): HARD. Someone
31
+ # checked; nobody has a mailbox there.
32
+ # - Any other lookalike (`gmail.co`, but also `ge.com` → me.com): a GUESS,
33
+ # so CONFIRMABLE. The error is added unless the record says the user
34
+ # confirmed the address as typed (Nondisposable::ConfirmableEmail). A
35
+ # real `ge.com` employee resubmits once and is never locked out.
36
+ #
37
+ # `true` and `:confirmable` mean the same thing. Before 0.5.0 `true` refused
38
+ # every lookalike outright, which measured against the Tranco top 1M would
39
+ # have refused staff at General Electric, EA and Aon (see CHANGELOG).
6
40
  def validate_each(record, attribute, value)
7
41
  return if value.blank?
8
42
 
9
43
  begin
10
- domain = value.to_s.split('@').last&.downcase
11
- return if domain.nil? # Invalid email format
44
+ # Without an "@" the whole value is judged as a domain, as it always
45
+ # has been — someone who typed only `tempmail.com` is still refused.
46
+ # Unicode form: DisposableDomain canonicalises it, and needs the
47
+ # unencoded spelling to match rows stored before 0.5.0.
48
+ domain = value.to_s.include?('@') ? Nondisposable::Domain.from_email(value, unicode: true) : Nondisposable::Domain.unicode(value)
49
+ return if domain.empty? # Invalid email format
50
+
51
+ config = Nondisposable.configuration
52
+
53
+ # A domain longer than DNS allows (RFC 1035; RFC 5321 caps it at
54
+ # 255) cannot receive mail and is in no list, so "not disposable"
55
+ # would be a lie: refuse it as invalid, whatever else is switched
56
+ # on. Only for real addresses — a value without "@" is `format:`'s.
57
+ if value.to_s.include?('@') && Nondisposable::Domain.too_long?(domain)
58
+ return reject(record, attribute, config.invalid_tld_error_message)
59
+ end
12
60
 
13
61
  if Nondisposable::DisposableDomain.disposable?(domain)
14
- record.errors.add(attribute, options[:message] || Nondisposable.configuration.error_message)
62
+ return reject(record, attribute, options[:message] || config.error_message)
63
+ end
64
+
65
+ return unless config.check_tld || config.reject_lookalike_domains
66
+ # No "@" at all is a malformed value, and this validator has never
67
+ # judged shape — that is what `format:` is for (see the README). Note
68
+ # `example@gmailmcom` DOES have one: a domain missing its dot is a real
69
+ # address at an impossible domain, which is very much our business.
70
+ return unless value.to_s.include?('@')
71
+ return unless Nondisposable::Tld.judgeable?(value)
72
+
73
+ # nil here means the domain has NO TLD (`example@gmailmcom` — a
74
+ # production-shaped typo that missed the dot), which fails "must end in a real TLD"
75
+ # just as surely as `.con` does.
76
+ tld = Nondisposable::Tld.extract(value)
77
+ impossible = tld.nil? || !Nondisposable::Tld.valid?(tld)
78
+ invalid = config.check_tld && impossible
79
+ blocked = config.check_tld && !invalid && Nondisposable::Tld.blocked?(tld)
80
+
81
+ if invalid || config.reject_lookalike_domains
82
+ suggestion = Nondisposable::Suggestion.for(value)
83
+ # `@gmial.com` has no local part to build "Did you mean ..." from,
84
+ # but it is still a reviewed typo: name the domain instead.
85
+ if suggestion.nil? && config.lookalike_distance.to_i.positive?
86
+ suggestion = Nondisposable::KnownTypos.correction_for(value)
87
+ end
88
+ if suggestion
89
+ message = config.lookalike_error_message.sub('%{suggestion}', suggestion)
90
+ # Hard: no confirmation can make an impossible TLD deliverable —
91
+ # whether or not the host turned check_tld on, `gmail.con` cannot
92
+ # receive the confirmation email — and a reviewed typo is not a
93
+ # guess. The error stays a plain message, exactly as before 0.5.0.
94
+ return reject(record, attribute, message) if impossible || Nondisposable::KnownTypos.include?(value)
95
+
96
+ unless Nondisposable::ConfirmableEmail.confirmed?(record, attribute)
97
+ # Typed, so a form can tell this error apart and offer the
98
+ # "yes, this is right" checkbox: errors.of_kind?(:email, :lookalike_domain).
99
+ record.errors.add(attribute, :lookalike_domain, message: message, suggestion: suggestion)
100
+ return
101
+ end
102
+ end
15
103
  end
104
+
105
+ return reject(record, attribute, config.invalid_tld_error_message) if invalid
106
+ return reject(record, attribute, config.blocked_tld_error_message) if blocked
16
107
  rescue StandardError => e
17
108
  if Nondisposable.configuration.on_check_failure == :reject
18
109
  Rails.logger.error "[nondisposable] Nondisposable validation error: #{e.message} — rejecting #{attribute} (on_check_failure = :reject)"
@@ -22,6 +113,13 @@ module ActiveModel
22
113
  end
23
114
  end
24
115
  end
116
+
117
+ private
118
+
119
+ def reject(record, attribute, message)
120
+ record.errors.add(attribute, message)
121
+ nil
122
+ end
25
123
  end
26
124
 
27
125
  module HelperMethods
@@ -0,0 +1,64 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Nondisposable
4
+ # The only lookalike domains this gem refuses outright.
5
+ #
6
+ # "One edit from gmail.com" is a guess, and offline a guess cannot tell
7
+ # `gmial.com` (a typo) from `ge.com` (General Electric, one edit from me.com).
8
+ # So a guess may only ever produce a CONFIRMABLE error — the user can say
9
+ # "yes, that is my address" and carry on. Refusing without that way out needs
10
+ # something stronger than a guess: a domain somebody looked at and found to be
11
+ # nothing but a misspelling of a provider — parked or for sale under the
12
+ # provider's misspelled name, redirecting to the provider, or squatted by a
13
+ # known typosquatter.
14
+ #
15
+ # Those reviewed domains live in data/known_typo_domains.txt, each with the
16
+ # correction it is a typo OF. The evidence behind every entry is in
17
+ # test/fixtures/lookalike_corpus.csv, and the test suite fails if an entry
18
+ # loses it. Nothing generates this list automatically; see the file's header
19
+ # for the bar an entry has to clear.
20
+ module KnownTypos
21
+ LIST_PATH = File.expand_path('../../data/known_typo_domains.txt', __dir__)
22
+
23
+ class << self
24
+ # Is this domain (or address) a reviewed typo?
25
+ def include?(email_or_domain)
26
+ !correction_for(email_or_domain).nil?
27
+ end
28
+
29
+ # The provider domain it is a typo of, or nil.
30
+ def correction_for(email_or_domain)
31
+ value = email_or_domain.to_s
32
+ domain = value.include?('@') ? Domain.from_email(value) : Domain.normalize(value)
33
+ return nil if domain.empty?
34
+ # A host who added a provider has told us its users are real, which
35
+ # outranks any review we did without knowing about them.
36
+ return nil if host_providers.include?(domain)
37
+
38
+ all[domain]
39
+ end
40
+
41
+ # typo => correction, frozen. Read once per process.
42
+ def all
43
+ @all ||= File.readlines(LIST_PATH, chomp: true).each_with_object({}) do |line, map|
44
+ value = line.sub(/#.*/, '').strip
45
+ next if value.empty?
46
+
47
+ typo, correction = value.split(/\s+/, 2)
48
+ map[Domain.normalize(typo)] = Domain.normalize(correction)
49
+ end.freeze
50
+ end
51
+
52
+ def reload!
53
+ @all = nil
54
+ self
55
+ end
56
+
57
+ private
58
+
59
+ def host_providers
60
+ Array(Nondisposable.configuration.additional_email_providers).map { |d| Domain.normalize(d) }
61
+ end
62
+ end
63
+ end
64
+ end
@@ -0,0 +1,209 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'set'
4
+
5
+ module Nondisposable
6
+ # Who wins when two lists disagree about a domain. ONE table, used both when
7
+ # an address is checked (#decide) and when the stored list is built
8
+ # (#filter), so the table and the check can never tell different stories:
9
+ #
10
+ # 1. config.excluded_domains (host) → allowed
11
+ # 2. data/never_disposable.txt (gem) → allowed real providers and ISPs
12
+ # 3. config.additional_domains (host) → blocked
13
+ # 4. data/privacy_relays.txt (gem) → config.block_privacy_relays decides
14
+ # 5. the merged upstream sources → blocked
15
+ #
16
+ # Before 0.6.0, additional_domains was checked FIRST, so a host could not
17
+ # exclude a domain it had also added. Now the host's exclusion wins: an
18
+ # allowlist entry is the more deliberate statement, and "a real person
19
+ # cannot sign up" is the costlier mistake.
20
+ #
21
+ # Row 2 outranking row 3 is the deliberate trade-off of this table: a host
22
+ # can no longer block a real provider (say `mail.ru`) through
23
+ # additional_domains. That list is for throwaway domains the upstream lists
24
+ # missed; refusing a whole real provider is a business rule, and belongs in
25
+ # the host's own validation where it can say so to the user.
26
+ #
27
+ # HOW EACH ROW MATCHES
28
+ #
29
+ # - excluded_domains: an exact match allows; an excluded parent can never be
30
+ # the REASON for a block, but a more specific listed subdomain still blocks
31
+ # (unchanged since 0.3.0).
32
+ # - never_disposable and privacy relays protect their whole namespace
33
+ # (`bak.rr.com` is covered by `rr.com`), whatever check_parent_domains says:
34
+ # upstream lists really do carry ISP subdomains like that one.
35
+ # - additional_domains and the stored list follow check_parent_domains.
36
+ # - No public suffix ever matches anything (Nondisposable::PublicSuffix).
37
+ module Precedence
38
+ NEVER_DISPOSABLE_PATH = File.expand_path('../../data/never_disposable.txt', __dir__)
39
+ PRIVACY_RELAYS_PATH = File.expand_path('../../data/privacy_relays.txt', __dir__)
40
+ EMAIL_PROVIDERS_PATH = File.expand_path('../../data/email_providers.txt', __dir__)
41
+
42
+ # A syntactically possible DNS name, after Domain.normalize: at least two
43
+ # labels of 1-63 characters, 253 in total, and a final label that is not
44
+ # all digits (that would be an IP address, not a domain). Underscores are
45
+ # tolerated inside labels — they are legal in DNS, if not in hostnames, and
46
+ # rejecting them would only ever make the list shorter, never safer.
47
+ LABEL = /\A[a-z0-9_](?:[a-z0-9_-]{0,61}[a-z0-9_])?\z/
48
+
49
+ class << self
50
+ # ---- Checking one address ---------------------------------------------
51
+
52
+ # The verdict for a normalized domain, given the stored-list candidates
53
+ # DisposableDomain.match_candidates produced for it:
54
+ #
55
+ # [:allow, nil] rows 1, 2 or 4 said no
56
+ # [:block, nil] rows 3 or 4 said yes
57
+ # [:lookup, candidates] ask the stored list (row 5) about these
58
+ def decide(domain, candidates)
59
+ config = Nondisposable.configuration
60
+ excluded = configured_set(config.excluded_domains)
61
+
62
+ return [:allow, nil] if excluded.include?(domain)
63
+ return [:allow, nil] if namespace_hit(domain, never_disposable)
64
+
65
+ remaining = candidates.reject { |candidate| excluded.include?(candidate) }
66
+
67
+ additional = configured_set(config.additional_domains)
68
+ return [:block, nil] if remaining.any? { |candidate| additional.include?(candidate) }
69
+
70
+ # The relay ALLOWANCE covers the whole relay namespace, whatever is
71
+ # excluded: adding an exclusion must never turn an allowed address into
72
+ # a refused one (filter drops every relay-namespace row the same way).
73
+ # Under block_privacy_relays an excluded relay name stops being a
74
+ # reason to block, like any other exclusion.
75
+ relay_names = namespace(domain).select { |name| privacy_relays.include?(name) }
76
+ if relay_names.any?
77
+ return [:allow, nil] unless config.block_privacy_relays
78
+ return [:block, nil] if relay_names.any? { |name| !excluded.include?(name) }
79
+ end
80
+
81
+ [:lookup, remaining]
82
+ end
83
+
84
+ # ---- Building the stored list -----------------------------------------
85
+
86
+ Result = Struct.new(:kept, :dropped) do
87
+ # { reason => [domains] }, for logging.
88
+ def dropped_by_reason
89
+ dropped.group_by { |_domain, reason| reason }.transform_values { |pairs| pairs.map(&:first).sort }
90
+ end
91
+ end
92
+
93
+ # Applies the table to a list about to be stored: every upstream domain
94
+ # is first put in Domain.normalize's canonical form, so every write path
95
+ # (fetch, seed, bundled fallback) stores exactly the name a lookup asks
96
+ # for, whoever the caller is. Each then goes through rows 1, 2 and 4 and
97
+ # the validity checks; the host's additional_domains are added through
98
+ # row 2 and the validity checks.
99
+ #
100
+ # Exclusions are exact here, as they always were: the stored list is not
101
+ # where parent-level exclusions take effect, #decide is.
102
+ #
103
+ # host_policy: false applies only the gem's own rules (validity, public
104
+ # suffixes, never-disposable, and relays always dropped), ignoring every
105
+ # host setting. The updater's size guards count that, so their numbers
106
+ # never depend on the host's configuration.
107
+ def filter(domains, host_policy: true)
108
+ config = Nondisposable.configuration
109
+ excluded = host_policy ? configured_set(config.excluded_domains) : Set.new
110
+ block_relays = host_policy && config.block_privacy_relays
111
+ kept = Set.new
112
+ dropped = []
113
+
114
+ domains.each do |raw|
115
+ domain = Domain.normalize(raw)
116
+ next if domain.empty?
117
+
118
+ reason = invalid_reason(domain) ||
119
+ (:excluded if excluded.include?(domain)) ||
120
+ (:never_disposable if namespace_hit(domain, never_disposable)) ||
121
+ (:privacy_relay if !block_relays && namespace_hit(domain, privacy_relays))
122
+ reason ? dropped << [domain, reason] : kept << domain
123
+ end
124
+
125
+ Array(host_policy ? config.additional_domains : []).each do |raw|
126
+ domain = Domain.normalize(raw)
127
+ next if domain.empty? || excluded.include?(domain)
128
+
129
+ reason = (invalid_reason(domain) && :invalid_additional_domain) ||
130
+ (:additional_domain_is_never_disposable if namespace_hit(domain, never_disposable))
131
+ reason ? dropped << [domain, reason] : kept << domain
132
+ end
133
+
134
+ Result.new(kept, dropped)
135
+ end
136
+
137
+ # nil when `domain` (normalized) may be stored, otherwise why not.
138
+ def invalid_reason(domain)
139
+ return :invalid unless hostname?(domain)
140
+ return :public_suffix if PublicSuffix.public_suffix?(domain)
141
+
142
+ nil
143
+ end
144
+
145
+ def hostname?(domain)
146
+ return false if domain.empty? || domain.length > 253
147
+
148
+ labels = domain.split('.', -1)
149
+ labels.size >= 2 &&
150
+ labels.all? { |label| label.length <= 63 && LABEL.match?(label) } &&
151
+ !labels.last.match?(/\A\d+\z/)
152
+ end
153
+
154
+ # ---- The bundled lists -------------------------------------------------
155
+
156
+ # data/never_disposable.txt + data/email_providers.txt, minus the relays.
157
+ # Unioned at runtime so a provider added for the typo checks is protected
158
+ # here too, without anybody remembering to copy it.
159
+ def never_disposable
160
+ @never_disposable ||= (read_list(NEVER_DISPOSABLE_PATH) | read_list(EMAIL_PROVIDERS_PATH)).subtract(privacy_relays).freeze
161
+ end
162
+
163
+ def privacy_relays
164
+ @privacy_relays ||= read_list(PRIVACY_RELAYS_PATH).freeze
165
+ end
166
+
167
+ def reload!
168
+ @never_disposable = nil
169
+ @privacy_relays = nil
170
+ self
171
+ end
172
+
173
+ # The domain and each of its parents down to its registrable domain:
174
+ # `a.b.rr.com` => ["a.b.rr.com", "b.rr.com", "rr.com"]. Empty for a public
175
+ # suffix. Never climbs past the registrable domain, so `sch.uk` is not in
176
+ # the namespace of `school.county.sch.uk`.
177
+ def namespace(domain)
178
+ registrable = PublicSuffix.registrable_domain(domain)
179
+ return [] unless registrable
180
+
181
+ labels = domain.split('.')
182
+ (0..(labels.size - registrable.count('.') - 1)).map { |i| labels.drop(i).join('.') }
183
+ end
184
+
185
+ # The most specific name in `domain`'s namespace that `set` contains.
186
+ def namespace_hit(domain, set)
187
+ namespace(domain).find { |name| set.include?(name) }
188
+ end
189
+
190
+ private
191
+
192
+ # Host-supplied, so normalized on every read rather than trusted.
193
+ def configured_set(values)
194
+ Array(values).each_with_object(Set.new) do |value, set|
195
+ domain = Domain.normalize(value)
196
+ set << domain unless domain.empty?
197
+ end
198
+ end
199
+
200
+ # One domain per line; `#` starts a comment, inline or whole-line.
201
+ def read_list(path)
202
+ File.foreach(path, encoding: 'UTF-8').each_with_object(Set.new) do |line, set|
203
+ domain = Domain.normalize(line.sub(/#.*/, ''))
204
+ set << domain unless domain.empty?
205
+ end
206
+ end
207
+ end
208
+ end
209
+ end
@@ -0,0 +1,147 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'set'
4
+
5
+ module Nondisposable
6
+ # Is this domain a PUBLIC SUFFIX — a name under which anybody can register
7
+ # their own domain, like `com`, `co.uk`, `com.br` or `github.io`?
8
+ #
9
+ # WHY THE BLOCKLIST NEEDS TO KNOW
10
+ #
11
+ # Parent-domain matching blocks `x.tempmail.com` because `tempmail.com` is on
12
+ # the list. That is exactly right for a registrable domain and exactly wrong
13
+ # for a public suffix: one bad line reading `com.br` in any upstream list
14
+ # would refuse every Brazilian company's staff, and nothing in a domain count
15
+ # or a file size would look unusual. So no public suffix is ever stored, ever
16
+ # accepted from `additional_domains`, and ever used as a parent-match
17
+ # candidate. That last one matters on its own: it keeps a row written by an
18
+ # older version of this gem from doing any damage either.
19
+ #
20
+ # WHY A BUNDLED SNAPSHOT AND NOT THE `public_suffix` GEM
21
+ #
22
+ # The same reason as Nondisposable::Tld: we need one yes/no answer, offline,
23
+ # on every signup, with no network and no extra runtime dependency. The list
24
+ # is the canonical file from https://publicsuffix.org/list/public_suffix_list.dat
25
+ # (MPL 2.0, redistributed unmodified), ~10,000 rules in ~330 KB, parsed once
26
+ # into frozen Sets. Both sections are honoured: the ICANN section (`co.uk`)
27
+ # and the private one (`github.io`, `blogspot.com`), because in both cases the
28
+ # names below belong to unrelated people. Refresh it with
29
+ # `rake nondisposable:psl:update` when cutting a release.
30
+ #
31
+ # STALENESS
32
+ #
33
+ # A suffix added after the snapshot is simply unknown: we would accept a
34
+ # blocklist entry for it, which is today's behaviour, so a stale snapshot can
35
+ # only ever be less protective, never lock anybody out.
36
+ module PublicSuffix
37
+ LIST_PATH = File.expand_path('../../data/public_suffix_list.dat', __dir__)
38
+
39
+ class << self
40
+ # True when `domain` is itself a public suffix. Single-label names (bare
41
+ # TLDs, known or not) always are: that is the PSL's implicit `*` rule.
42
+ # Expects a normalized domain (see Nondisposable::Domain.normalize).
43
+ def public_suffix?(domain)
44
+ domain = domain.to_s
45
+ return false if domain.empty?
46
+
47
+ labels = domain.split('.')
48
+ return true if labels.size == 1
49
+
50
+ list = rules
51
+ return false if list[:exceptions].include?(domain)
52
+ return true if list[:exact].include?(domain)
53
+
54
+ list[:wildcards].include?(labels.drop(1).join('.'))
55
+ end
56
+
57
+ # The registrable domain `domain` belongs to: its longest public suffix
58
+ # plus one label. `school.county.sch.uk` => itself (`county.sch.uk` is a
59
+ # suffix via `*.sch.uk`); `x.tempmail.com` => `tempmail.com`. nil when
60
+ # `domain` is itself a public suffix.
61
+ #
62
+ # Everything ABOVE this name belongs to the registry or to unrelated
63
+ # registrants, so no list entry up there may ever match `domain` — not
64
+ # even a name like `sch.uk` that is not itself a suffix but sits above a
65
+ # wildcard one.
66
+ def registrable_domain(domain)
67
+ labels = domain.to_s.split('.')
68
+ return nil if labels.empty? || public_suffix?(domain.to_s)
69
+
70
+ # An exception rule prevails (`!city.kawasaki.jp` makes its parent the
71
+ # suffix); otherwise the longest suffix that is public wins.
72
+ exceptions = rules[:exceptions]
73
+ exception = (1..labels.size).find { |n| exceptions.include?(labels.last(n).join('.')) }
74
+ suffix_size = if exception
75
+ exception - 1
76
+ else
77
+ (1...labels.size).select { |n| public_suffix?(labels.last(n).join('.')) }.max || 1
78
+ end
79
+ labels.last(suffix_size + 1).join('.')
80
+ end
81
+
82
+ # IANA-style provenance for health checks: "2026-10-16_23-32-13_UTC".
83
+ def version
84
+ @version ||= begin
85
+ line = File.foreach(LIST_PATH).first(20).find { |l| l.start_with?('// VERSION:') }
86
+ line&.delete_prefix('// VERSION:')&.strip
87
+ rescue StandardError
88
+ nil
89
+ end
90
+ end
91
+
92
+ def rule_count
93
+ list = rules
94
+ list[:exact].size + list[:wildcards].size + list[:exceptions].size
95
+ end
96
+
97
+ # Reset the memoized rules. Called by the updater after it rewrites the
98
+ # file, and by tests.
99
+ def reload!
100
+ @rules = nil
101
+ @version = nil
102
+ self
103
+ end
104
+
105
+ # Parses PSL text into the three rule kinds. Public so the updater can
106
+ # judge a downloaded body before it replaces the bundled one.
107
+ #
108
+ # Rules are stored in their ASCII (punycode) form, because that is the
109
+ # form Domain.normalize produces. A unicode rule we cannot convert is
110
+ # kept as-is: it then simply never matches, which errs towards today's
111
+ # behaviour rather than towards refusing anybody.
112
+ def parse(text)
113
+ exact = Set.new
114
+ wildcards = Set.new
115
+ exceptions = Set.new
116
+
117
+ text.each_line do |line|
118
+ rule = line.strip
119
+ next if rule.empty? || rule.start_with?('//')
120
+
121
+ rule = rule.split(/\s/, 2).first # the PSL ends a rule at whitespace
122
+ if rule.start_with?('!')
123
+ exceptions << ascii(rule.delete_prefix('!'))
124
+ elsif rule.start_with?('*.')
125
+ wildcards << ascii(rule.delete_prefix('*.'))
126
+ else
127
+ exact << ascii(rule)
128
+ end
129
+ end
130
+
131
+ { exact: exact.freeze, wildcards: wildcards.freeze, exceptions: exceptions.freeze }.freeze
132
+ end
133
+
134
+ private
135
+
136
+ def rules
137
+ @rules ||= parse(File.read(LIST_PATH, encoding: 'UTF-8'))
138
+ end
139
+
140
+ def ascii(rule)
141
+ Nondisposable::Domain.normalize(rule)
142
+ rescue StandardError
143
+ rule.downcase
144
+ end
145
+ end
146
+ end
147
+ end
@@ -0,0 +1,88 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'net/http'
4
+ require 'uri'
5
+
6
+ module Nondisposable
7
+ # Refreshes the vendored Public Suffix List snapshot.
8
+ #
9
+ # ⚠️ A MAINTENANCE task, like TldListUpdater: it rewrites a file inside the
10
+ # gem, so run it in a checkout before cutting a release:
11
+ #
12
+ # rake nondisposable:psl:update
13
+ #
14
+ # Host apps never need it. A stale PSL can only be LESS protective (a suffix
15
+ # created after the snapshot is not known to be one), never lock anyone out.
16
+ class PublicSuffixListUpdater
17
+ # The PSL maintainers ask to be fetched from here and only here.
18
+ LIST_URL = 'https://publicsuffix.org/list/public_suffix_list.dat'
19
+
20
+ OPEN_TIMEOUT = 10
21
+ READ_TIMEOUT = 10
22
+
23
+ # The list holds ~10,000 rules. A body with a fraction of that is an error
24
+ # page or a truncated download, not the internet shrinking.
25
+ MINIMUM_PLAUSIBLE_RULE_COUNT = 5_000
26
+
27
+ class << self
28
+ # Returns the number of rules written, or nil if nothing was written.
29
+ def update(path: Nondisposable::PublicSuffix::LIST_PATH)
30
+ log :info, "Fetching the Public Suffix List from #{LIST_URL}..."
31
+
32
+ response = fetch
33
+ unless response.is_a?(Net::HTTPSuccess)
34
+ log :error, "Failed to download the Public Suffix List. HTTP status: #{response.code}. Keeping the existing snapshot."
35
+ return nil
36
+ end
37
+
38
+ body = response.body.to_s.dup.force_encoding(Encoding::UTF_8)
39
+ rules = Nondisposable::PublicSuffix.parse(body)
40
+ count = rules.values.sum(&:size)
41
+
42
+ unless body.include?('===BEGIN ICANN DOMAINS===') && count >= MINIMUM_PLAUSIBLE_RULE_COUNT
43
+ log :error, "Refusing to write a Public Suffix List with only #{count} rules (expected at least #{MINIMUM_PLAUSIBLE_RULE_COUNT} and an ICANN section). Keeping the existing snapshot."
44
+ return nil
45
+ end
46
+
47
+ # Written whole, then moved into place, so an interrupted run can never
48
+ # leave half a list behind.
49
+ tmp = "#{path}.tmp"
50
+ File.write(tmp, body)
51
+ File.rename(tmp, path)
52
+ Nondisposable::PublicSuffix.reload!
53
+
54
+ log :info, "Wrote #{count} rules to #{path} (#{Nondisposable::PublicSuffix.version})."
55
+ count
56
+ rescue StandardError => e
57
+ log :error, "Could not update the Public Suffix List: #{e.class}: #{e.message}. Keeping the existing snapshot."
58
+ nil
59
+ ensure
60
+ File.delete(tmp) if tmp && File.exist?(tmp)
61
+ end
62
+
63
+ private
64
+
65
+ def fetch
66
+ uri = URI(LIST_URL)
67
+ Net::HTTP.start(
68
+ uri.host,
69
+ uri.port,
70
+ use_ssl: uri.scheme == 'https',
71
+ open_timeout: OPEN_TIMEOUT,
72
+ read_timeout: READ_TIMEOUT
73
+ ) { |http| http.get(uri.request_uri) }
74
+ end
75
+
76
+ # Usable from a bare `rake` run in a gem checkout, where there is no Rails
77
+ # logger to talk to.
78
+ def log(level, message)
79
+ message = "[nondisposable] #{message}"
80
+ if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
81
+ Rails.logger.public_send(level, message)
82
+ else
83
+ warn message
84
+ end
85
+ end
86
+ end
87
+ end
88
+ end