nondisposable 0.3.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +73 -0
- data/README.md +305 -3
- data/Rakefile +5 -0
- data/app/models/nondisposable/disposable_domain.rb +53 -34
- data/app/models/nondisposable/source_snapshot.rb +66 -0
- data/data/email_providers.txt +276 -0
- data/data/iana_tlds.txt +1439 -0
- data/data/known_typo_domains.txt +99 -0
- data/data/never_disposable.txt +480 -0
- data/data/privacy_relays.txt +42 -0
- data/data/public_suffix_list.dat +16571 -0
- data/gemfiles/rails_7.2.gemfile +1 -0
- data/gemfiles/rails_8.0.gemfile +1 -0
- data/gemfiles/rails_8.1.gemfile +1 -0
- data/lib/generators/nondisposable/install_generator.rb +1 -0
- data/lib/generators/nondisposable/templates/create_nondisposable_source_snapshots.rb.erb +42 -0
- data/lib/generators/nondisposable/templates/nondisposable.rb +67 -1
- data/lib/generators/nondisposable/upgrade_generator.rb +43 -0
- data/lib/nondisposable/confirmable_email.rb +75 -0
- data/lib/nondisposable/domain.rb +238 -0
- data/lib/nondisposable/domain_list_updater.rb +510 -88
- data/lib/nondisposable/email_validator.rb +101 -3
- data/lib/nondisposable/known_typos.rb +64 -0
- data/lib/nondisposable/precedence.rb +209 -0
- data/lib/nondisposable/public_suffix.rb +147 -0
- data/lib/nondisposable/public_suffix_list_updater.rb +88 -0
- data/lib/nondisposable/source.rb +233 -0
- data/lib/nondisposable/source_fetcher.rb +69 -0
- data/lib/nondisposable/source_merge.rb +127 -0
- data/lib/nondisposable/suggestion.rb +336 -0
- data/lib/nondisposable/tld.rb +190 -0
- data/lib/nondisposable/tld_list_updater.rb +101 -0
- data/lib/nondisposable/version.rb +1 -1
- data/lib/nondisposable.rb +153 -6
- data/lib/tasks/nondisposable.rake +66 -0
- metadata +24 -2
|
@@ -0,0 +1,336 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Nondisposable
|
|
4
|
+
# "Did you mean gmail.com?"
|
|
5
|
+
#
|
|
6
|
+
# A TLD check catches the typos that produce a domain ending which cannot
|
|
7
|
+
# exist — `.con`, `.cpm`, `.ocm`. It structurally cannot catch the far more
|
|
8
|
+
# common ones, because `.co` (Colombia), `.cm` (Cameroon) and `.om` (Oman) are
|
|
9
|
+
# all real, delegated TLDs, as are `.se` and `.es`. `user@gmail.co` is a
|
|
10
|
+
# perfectly well-formed address at a perfectly real TLD, and it is still
|
|
11
|
+
# almost always a finger that slipped off the `m`.
|
|
12
|
+
#
|
|
13
|
+
# So this layer asks a different question: is this domain one keystroke away
|
|
14
|
+
# from a well-known email provider, while not being one itself?
|
|
15
|
+
#
|
|
16
|
+
# HOW IT DECIDES
|
|
17
|
+
#
|
|
18
|
+
# Optimal string alignment distance (Damerau-Levenshtein restricted to
|
|
19
|
+
# adjacent transpositions) against data/email_providers.txt. Transposition
|
|
20
|
+
# matters more than it looks: `gmial.com` is the single most common Gmail
|
|
21
|
+
# misspelling, and plain Levenshtein scores it 2 while a human sees one
|
|
22
|
+
# mistake. Under OSA it scores 1, alongside `gmai.com`, `gmail.co` and
|
|
23
|
+
# `gmail.con`.
|
|
24
|
+
#
|
|
25
|
+
# WHAT ONE EDIT CANNOT TELL APART, AND THE TWO FILTERS THAT FOLLOW
|
|
26
|
+
#
|
|
27
|
+
# Replaying "one edit from a provider" against the Tranco top 1M (2026-10-17,
|
|
28
|
+
# docs/evidence/) flagged 707 domains, and a lot of them are large real
|
|
29
|
+
# organisations: ge.com, mi.com and se.com are one edit from me.com; ea.com and
|
|
30
|
+
# aa.com from ya.com; bol.com and uol.com from aol.com; grail.com from
|
|
31
|
+
# gmail.com. Distance alone cannot see the difference, so two filters decide
|
|
32
|
+
# what is worth suggesting at all:
|
|
33
|
+
#
|
|
34
|
+
# 1. SHORT NAMES ONLY TAKE EDITS AFTER THE NAME. When a provider's name is
|
|
35
|
+
# four characters or fewer (me, ya, qq, aol, msn, mail, live, zoho…),
|
|
36
|
+
# one edit inside the name lands on somebody else's real domain far more
|
|
37
|
+
# often than on a typo. `me.cmo` and `live.cm` are still suggested;
|
|
38
|
+
# `ge.com`, `mil.ru` and `love.com` are not.
|
|
39
|
+
# 2. EVERY EDIT MUST LOOK LIKE A FINGER. A swap (`gmial`), a dropped letter
|
|
40
|
+
# (`gmal`, `gmail.co`), a doubled letter (`gmmail`), an extra vowel
|
|
41
|
+
# (`gamail`) or a neighbouring key on QWERTY (`gnail`, `gmail.con`) is
|
|
42
|
+
# how people mistype. A letter from across the keyboard (`grail`,
|
|
43
|
+
# `pmail`, `hive`) is how people spell a different word, and the same
|
|
44
|
+
# holds after the name: `.au` is not a slip for `.ar`, nor `.dev` for
|
|
45
|
+
# `.de`. Dropping the FIRST letter is not counted either — `cloud.com`
|
|
46
|
+
# and `aver.com` are what that spells.
|
|
47
|
+
#
|
|
48
|
+
# Measured on the 70% tuning split of test/fixtures/lookalike_corpus.csv
|
|
49
|
+
# (precision over typo+real rows): distance alone 0.07, filter 1 alone 0.27,
|
|
50
|
+
# both 0.55, at 79% recall of reviewed typos. The corpus is the Tranco top 1M,
|
|
51
|
+
# so it is crowded with real popular domains — a signup form sees far more
|
|
52
|
+
# typos per lookalike than this does. The other 30% is a RETROSPECTIVE
|
|
53
|
+
# holdout, measured by test/nondisposable/lookalike_corpus_test.rb: the hash
|
|
54
|
+
# split was chosen before tuning but only committed with the tuned filters,
|
|
55
|
+
# so it is not a pre-registered holdout, and one early draft was seen
|
|
56
|
+
# against it.
|
|
57
|
+
#
|
|
58
|
+
# These filters only decide what to SUGGEST. They are a likelihood, not proof,
|
|
59
|
+
# and nothing in this file ever justifies refusing an address on its own —
|
|
60
|
+
# see Nondisposable::KnownTypos for the only domains the gem refuses outright,
|
|
61
|
+
# and ActiveModel::Validations::NondisposableValidator for how a suggestion
|
|
62
|
+
# becomes a confirmable error rather than a wall.
|
|
63
|
+
#
|
|
64
|
+
# WHY THE DEFAULT THRESHOLD IS 1, AND WHY THIS IS OFF BY DEFAULT
|
|
65
|
+
#
|
|
66
|
+
# One edit is the distance at which a guess is worth showing; at two,
|
|
67
|
+
# legitimately distinct domains start colliding. The exact match check runs
|
|
68
|
+
# first and always wins, which is why the provider list has to be generous —
|
|
69
|
+
# see the header of data/email_providers.txt.
|
|
70
|
+
#
|
|
71
|
+
# Suggesting is always available and never blocks:
|
|
72
|
+
#
|
|
73
|
+
# Nondisposable.suggestion_for("someone@gmial.com") # => "someone@gmail.com"
|
|
74
|
+
# Nondisposable.suggestion_for("someone@gmail.com") # => nil
|
|
75
|
+
module Suggestion
|
|
76
|
+
LIST_PATH = File.expand_path('../../data/email_providers.txt', __dir__)
|
|
77
|
+
|
|
78
|
+
# A provider name this short only takes edits after the name. See filter 1
|
|
79
|
+
# in the header for the evidence.
|
|
80
|
+
SHORT_NAME_LENGTH = 4
|
|
81
|
+
|
|
82
|
+
VOWELS = 'aeiou'
|
|
83
|
+
|
|
84
|
+
# Physical neighbours on a QWERTY keyboard, the layout of Spain, Latin
|
|
85
|
+
# America and the English-speaking world (and the default on phones there).
|
|
86
|
+
# `ñ` sits right of `l` on the Spanish layout. AZERTY and QWERTZ are left out
|
|
87
|
+
# on purpose: unioning them makes every key a neighbour of more keys, and
|
|
88
|
+
# QWERTZ's z/y swap alone would start suggesting zoho.com to zozo.com.
|
|
89
|
+
KEYBOARD_ROWS = %w[1234567890 qwertyuiop asdfghjklñ zxcvbnm].freeze
|
|
90
|
+
|
|
91
|
+
class << self
|
|
92
|
+
# The full corrected address, or nil when we have nothing useful to say.
|
|
93
|
+
def for(email)
|
|
94
|
+
email = email.to_s
|
|
95
|
+
# Bytes that are not UTF-8 have no keystrokes to correct.
|
|
96
|
+
email = email.dup.force_encoding(Encoding::UTF_8) unless email.encoding == Encoding::UTF_8
|
|
97
|
+
return nil unless email.valid_encoding?
|
|
98
|
+
|
|
99
|
+
email = email.strip
|
|
100
|
+
local, _, domain = email.rpartition('@')
|
|
101
|
+
return nil if local.empty? || domain.empty?
|
|
102
|
+
|
|
103
|
+
corrected = correct_domain(domain)
|
|
104
|
+
return nil if corrected.nil?
|
|
105
|
+
|
|
106
|
+
"#{local}@#{corrected}"
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# The corrected DOMAIN alone, or nil. Split out so a host can offer
|
|
110
|
+
# "did you mean …?" next to a domain field, not just an email field.
|
|
111
|
+
def correct_domain(domain)
|
|
112
|
+
typed = Domain.unicode(domain)
|
|
113
|
+
return nil if typed.empty?
|
|
114
|
+
# A real provider is never a typo of another real provider.
|
|
115
|
+
return nil if providers.include?(Domain.normalize(typed))
|
|
116
|
+
# 0 switches the whole layer off, reviewed list included.
|
|
117
|
+
return nil if Nondisposable.configuration.lookalike_distance.to_i < 1
|
|
118
|
+
|
|
119
|
+
# A reviewed typo carries its own correction, and it wins over any
|
|
120
|
+
# guess: the list exists precisely because we checked it.
|
|
121
|
+
KnownTypos.correction_for(typed) || guess(typed)
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# The ranker alone: distance plus the two filters, without the reviewed
|
|
125
|
+
# list. Public so the corpus test can measure the guess on its own —
|
|
126
|
+
# measuring it with the list in front would mostly measure the list.
|
|
127
|
+
def guess(domain)
|
|
128
|
+
typed = Domain.unicode(domain)
|
|
129
|
+
return nil if typed.empty?
|
|
130
|
+
return nil if providers.include?(Domain.normalize(typed))
|
|
131
|
+
|
|
132
|
+
# Punycode letters are an encoding, not keystrokes: `xn--gmal-xpa.com`
|
|
133
|
+
# is somebody typing `gmaíl.com`, and its distance to gmail.com in
|
|
134
|
+
# ASCII is meaningless. We judge the unicode form when we get it and
|
|
135
|
+
# stay silent on the encoded one.
|
|
136
|
+
return nil if typed.encoding == Encoding::BINARY # not UTF-8: nothing typed to measure
|
|
137
|
+
return nil if typed.split('.').any? { |label| label.start_with?('xn--') }
|
|
138
|
+
|
|
139
|
+
threshold = Nondisposable.configuration.lookalike_distance.to_i
|
|
140
|
+
return nil if threshold < 1
|
|
141
|
+
|
|
142
|
+
best = nil
|
|
143
|
+
best_distance = threshold + 1
|
|
144
|
+
|
|
145
|
+
targets.each do |candidate|
|
|
146
|
+
# Cheap rejection before the O(n*m) walk: an edit changes the length
|
|
147
|
+
# by at most 1 per operation, so anything further apart than the
|
|
148
|
+
# threshold in length alone cannot be within it.
|
|
149
|
+
next if (candidate.length - typed.length).abs > threshold
|
|
150
|
+
|
|
151
|
+
distance = osa_distance(typed, candidate, best_distance)
|
|
152
|
+
next if distance >= best_distance
|
|
153
|
+
next unless plausible?(typed, candidate, distance)
|
|
154
|
+
|
|
155
|
+
best = candidate
|
|
156
|
+
best_distance = distance
|
|
157
|
+
break if distance == 1 # nothing plausible can beat one edit; stop early
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
best
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# Every provider, as a frozen Set of canonical (punycode) names: the
|
|
164
|
+
# bundled list plus anything the host added. This is the EXEMPTION set —
|
|
165
|
+
# an exact match here is never called a typo. Memoised per configuration
|
|
166
|
+
# so a host changing config in a test or console is picked up.
|
|
167
|
+
def providers
|
|
168
|
+
refresh_cache
|
|
169
|
+
@providers
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
# The same providers as a human would type them (unicode, not punycode):
|
|
173
|
+
# what a keystroke is measured against. `bücher.d` is one edit from
|
|
174
|
+
# `bücher.de` and eleven from `xn--bcher-kva.de`.
|
|
175
|
+
def targets
|
|
176
|
+
refresh_cache
|
|
177
|
+
@targets
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
def reload!
|
|
181
|
+
@providers = nil
|
|
182
|
+
@targets = nil
|
|
183
|
+
@bundled_providers = nil
|
|
184
|
+
@cache_key = nil
|
|
185
|
+
self
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
private
|
|
189
|
+
|
|
190
|
+
def refresh_cache
|
|
191
|
+
extra = Array(Nondisposable.configuration.additional_email_providers).map { |d| Domain.unicode(d) }.reject(&:empty?)
|
|
192
|
+
cache_key = extra.hash
|
|
193
|
+
return if @cache_key == cache_key && @providers && @targets
|
|
194
|
+
|
|
195
|
+
@targets = (bundled_providers + extra).uniq.freeze
|
|
196
|
+
@providers = @targets.map { |d| Domain.normalize(d) }.to_set.freeze
|
|
197
|
+
@cache_key = cache_key
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
def bundled_providers
|
|
201
|
+
@bundled_providers ||= File.readlines(LIST_PATH, chomp: true).filter_map do |line|
|
|
202
|
+
value = line.strip.downcase
|
|
203
|
+
value unless value.empty? || value.start_with?('#')
|
|
204
|
+
end
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
# Does the difference between what was typed and this provider look like
|
|
208
|
+
# a slip of the finger? The two filters from the header, in code.
|
|
209
|
+
def plausible?(typed, candidate, distance)
|
|
210
|
+
name = candidate.split('.').first
|
|
211
|
+
# The dot itself missed or swapped for a letter (`gmailmcom`,
|
|
212
|
+
# `gmailcom`): nobody owns a domain shaped like that, and the TLD path
|
|
213
|
+
# refuses it anyway, so it is always worth naming the correction.
|
|
214
|
+
return true if typed.count('.') < candidate.count('.') && typed.start_with?(name)
|
|
215
|
+
|
|
216
|
+
inside_name = typed.split('.').first != name
|
|
217
|
+
return false if inside_name && name.length <= SHORT_NAME_LENGTH
|
|
218
|
+
# The edit model below describes ONE edit. A host that widened
|
|
219
|
+
# lookalike_distance has opted into coarser guesses; filter 1 still
|
|
220
|
+
# applies, filter 2 has nothing to say.
|
|
221
|
+
return true if distance > 1
|
|
222
|
+
|
|
223
|
+
finger_slip?(typed, candidate)
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
# One edit, classified — in the name or after it, the same fingers type
|
|
227
|
+
# both (`gmail.cpm` is as plausible as `gmsil.com`, `yahoo.com.au` as
|
|
228
|
+
# implausible as `grail.com`). Callers have already established the
|
|
229
|
+
# distance is exactly 1, so exactly one of these shapes applies.
|
|
230
|
+
def finger_slip?(typed, candidate)
|
|
231
|
+
i = 0
|
|
232
|
+
i += 1 while i < typed.length && i < candidate.length && typed[i] == candidate[i]
|
|
233
|
+
|
|
234
|
+
case typed.length - candidate.length
|
|
235
|
+
when -1
|
|
236
|
+
# A dropped letter (`gmal`, `hotmal`, `gmail.co`) — but not the
|
|
237
|
+
# first one: `cloud.com`, `aver.com` and `mail.ch` are what
|
|
238
|
+
# dropping the first letter of icloud, naver and gmail.ch spells,
|
|
239
|
+
# and every one of them is somebody's real domain.
|
|
240
|
+
i.positive?
|
|
241
|
+
when 1
|
|
242
|
+
# An extra letter is a slip when it repeats, or sits next to, a
|
|
243
|
+
# letter it was typed beside (`gmmail`, `gmaiul` — not `mlive`), or
|
|
244
|
+
# when it is a vowel: spelling by ear adds vowels (`gamail`,
|
|
245
|
+
# `gimail`) far more often than consonants.
|
|
246
|
+
extra = typed[i]
|
|
247
|
+
return true if VOWELS.include?(extra)
|
|
248
|
+
|
|
249
|
+
beside = [i.positive? ? typed[i - 1] : nil, typed[i + 1]].compact
|
|
250
|
+
beside.any? { |near| near == extra || neighbours?(near, extra) }
|
|
251
|
+
else
|
|
252
|
+
# A swap (`gmial`) is the one mistake OSA counts as one edit…
|
|
253
|
+
return true if typed[i + 1] == candidate[i] && typed[i] == candidate[i + 1]
|
|
254
|
+
|
|
255
|
+
# …otherwise it is a substitution, which has to be a neighbouring
|
|
256
|
+
# key — or the same key with an accent (`gmaíl`: on a Spanish
|
|
257
|
+
# keyboard í is i after a dead key, not a different letter).
|
|
258
|
+
same_key?(typed[i], candidate[i]) || neighbours?(typed[i], candidate[i])
|
|
259
|
+
end
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
def same_key?(a, b)
|
|
263
|
+
a.unicode_normalize(:nfd)[0] == b.unicode_normalize(:nfd)[0]
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
def neighbours?(a, b)
|
|
267
|
+
keyboard.fetch(a, nil)&.include?(b) || false
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
# key => the keys around it, on a staggered QWERTY: the row's own left and
|
|
271
|
+
# right, and the two keys above and below that it physically touches.
|
|
272
|
+
def keyboard
|
|
273
|
+
@keyboard ||= begin
|
|
274
|
+
map = Hash.new { |h, k| h[k] = Set.new }
|
|
275
|
+
KEYBOARD_ROWS.each_with_index do |row, r|
|
|
276
|
+
row.each_char.with_index do |key, c|
|
|
277
|
+
touching = [[r, c - 1], [r, c + 1], [r - 1, c], [r - 1, c + 1], [r + 1, c - 1], [r + 1, c]]
|
|
278
|
+
touching.each do |rr, cc|
|
|
279
|
+
next if rr.negative? || cc.negative? || rr >= KEYBOARD_ROWS.size
|
|
280
|
+
|
|
281
|
+
other = KEYBOARD_ROWS[rr][cc]
|
|
282
|
+
map[key] << other if other
|
|
283
|
+
end
|
|
284
|
+
end
|
|
285
|
+
end
|
|
286
|
+
map.default_proc = nil
|
|
287
|
+
map.freeze
|
|
288
|
+
end
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
# Optimal string alignment distance, with an early bail-out.
|
|
292
|
+
#
|
|
293
|
+
# Two rows instead of a full matrix (we only ever need the previous two),
|
|
294
|
+
# and if the best cell in a row already exceeds `cutoff` no later row can
|
|
295
|
+
# come back under it — the distance only grows — so we stop. With ~250
|
|
296
|
+
# candidates per lookup and a threshold of 1, almost every candidate exits
|
|
297
|
+
# on the length check above or within the first row or two.
|
|
298
|
+
# (Empty inputs need no special case: with an empty `a` the loop never
|
|
299
|
+
# runs and the seeded row already holds the right answer, b.length.)
|
|
300
|
+
def osa_distance(a, b, cutoff)
|
|
301
|
+
prev_prev = nil
|
|
302
|
+
prev = (0..b.length).to_a
|
|
303
|
+
current = Array.new(b.length + 1)
|
|
304
|
+
|
|
305
|
+
a.each_char.with_index do |a_char, i|
|
|
306
|
+
current[0] = i + 1
|
|
307
|
+
row_min = current[0]
|
|
308
|
+
|
|
309
|
+
b.each_char.with_index do |b_char, j|
|
|
310
|
+
cost = a_char == b_char ? 0 : 1
|
|
311
|
+
value = [
|
|
312
|
+
current[j] + 1, # insertion
|
|
313
|
+
prev[j + 1] + 1, # deletion
|
|
314
|
+
prev[j] + cost # substitution
|
|
315
|
+
].min
|
|
316
|
+
|
|
317
|
+
# Transposition: the one that makes `gmial` a single mistake.
|
|
318
|
+
if i.positive? && j.positive? && a_char == b[j - 1] && a[i - 1] == b_char
|
|
319
|
+
value = [value, prev_prev[j - 1] + cost].min
|
|
320
|
+
end
|
|
321
|
+
|
|
322
|
+
current[j + 1] = value
|
|
323
|
+
row_min = value if value < row_min
|
|
324
|
+
end
|
|
325
|
+
|
|
326
|
+
return cutoff if row_min >= cutoff
|
|
327
|
+
|
|
328
|
+
prev_prev = prev
|
|
329
|
+
prev = current.dup
|
|
330
|
+
end
|
|
331
|
+
|
|
332
|
+
prev[b.length]
|
|
333
|
+
end
|
|
334
|
+
end
|
|
335
|
+
end
|
|
336
|
+
end
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'set'
|
|
4
|
+
|
|
5
|
+
module Nondisposable
|
|
6
|
+
# Is the bit after the last dot a real top-level domain?
|
|
7
|
+
#
|
|
8
|
+
# This is a different question from "is this a disposable provider", and it
|
|
9
|
+
# catches a different kind of bad address: the typo. `user@gmail.con` is not a
|
|
10
|
+
# throwaway — it is a real person who will never receive their confirmation
|
|
11
|
+
# email, because `.con` does not exist and never has. Those accounts are born
|
|
12
|
+
# dead: nobody can reach the user, and the user cannot recover the account.
|
|
13
|
+
#
|
|
14
|
+
# WHY A BUNDLED LIST AND NOT A DEPENDENCY
|
|
15
|
+
#
|
|
16
|
+
# The list is the IANA root zone database — the authoritative register of every
|
|
17
|
+
# delegated TLD on the internet, ~1,438 entries in about 9 KB:
|
|
18
|
+
#
|
|
19
|
+
# https://data.iana.org/TLD/tlds-alpha-by-domain.txt
|
|
20
|
+
#
|
|
21
|
+
# Every "TLD list" repository on GitHub is a scrape of that file, so we go to
|
|
22
|
+
# the source. The `tld` gem last shipped in 2014 and its list predates roughly
|
|
23
|
+
# 1,200 of today's TLDs. `public_suffix` is excellent and alive, but it answers
|
|
24
|
+
# "is this a valid public suffix" (which includes private entries like
|
|
25
|
+
# `github.io`) and it would pull the whole Public Suffix List in as a runtime
|
|
26
|
+
# dependency — the same trade-off DisposableDomain::PARENT_MATCH_DEPTH already
|
|
27
|
+
# declined, for the same reason.
|
|
28
|
+
#
|
|
29
|
+
# WHY IN MEMORY AND NOT IN THE DATABASE
|
|
30
|
+
#
|
|
31
|
+
# Disposable domains live in a table because there are 8,000+ of them and they
|
|
32
|
+
# change every few days. TLDs are 1,438 strings that change a handful of times
|
|
33
|
+
# a year. A frozen Set costs ~100 KB of process memory, answers in O(1) with no
|
|
34
|
+
# query per signup, needs no migration, and — unlike a table — cannot be empty
|
|
35
|
+
# on a fresh install. Refresh it with `rake nondisposable:tlds:update` (or
|
|
36
|
+
# `Nondisposable::TldListUpdater.update`) when cutting a release.
|
|
37
|
+
#
|
|
38
|
+
# STALENESS, AND WHY IT CANNOT LOCK ANYONE OUT
|
|
39
|
+
#
|
|
40
|
+
# The failure mode of a stale list is rejecting somebody whose TLD was
|
|
41
|
+
# delegated after the snapshot. `Nondisposable.configuration.additional_tlds`
|
|
42
|
+
# is the escape hatch: a host can accept a brand-new TLD immediately, without
|
|
43
|
+
# waiting for a gem release. And the check is opt-in (`config.check_tld`), so
|
|
44
|
+
# upgrading the gem never silently starts rejecting anybody.
|
|
45
|
+
module Tld
|
|
46
|
+
# The vendored IANA snapshot. Its first line is IANA's own version header,
|
|
47
|
+
# kept verbatim so the provenance and date of the data ship with it.
|
|
48
|
+
LIST_PATH = File.expand_path('../../data/iana_tlds.txt', __dir__)
|
|
49
|
+
|
|
50
|
+
# The names RFC 6761 and RFC 2606 reserve so they can NEVER be delegated.
|
|
51
|
+
# They are therefore absent from the root zone, and `check_tld` rejects
|
|
52
|
+
# them — which is right for a signup form, because no human types
|
|
53
|
+
# `me@home.test`, and an address there could never receive the confirmation
|
|
54
|
+
# email anyway.
|
|
55
|
+
#
|
|
56
|
+
# It is also, on the day you switch `check_tld` on, why half your test suite
|
|
57
|
+
# goes red: fixtures live at `user@example.test` for exactly the same reason
|
|
58
|
+
# the names are reserved. That is a configuration question, not a bug, and
|
|
59
|
+
# this constant is here so the answer reads like a sentence:
|
|
60
|
+
#
|
|
61
|
+
# config.additional_tlds = Nondisposable::Tld::SPECIAL_USE if Rails.env.local?
|
|
62
|
+
#
|
|
63
|
+
# Scope it to your non-production environments. Somewhere in the world
|
|
64
|
+
# somebody is running an app that really does deliver mail inside `.local`;
|
|
65
|
+
# if that is you, add it in production too and you are the exception that
|
|
66
|
+
# proves why this is configuration.
|
|
67
|
+
SPECIAL_USE = %w[test example invalid localhost local onion].freeze
|
|
68
|
+
|
|
69
|
+
class << self
|
|
70
|
+
# Every TLD IANA has delegated, downcased, as a frozen Set.
|
|
71
|
+
# Memoized: read once per process, never re-read.
|
|
72
|
+
def all
|
|
73
|
+
@all ||= parse(File.readlines(LIST_PATH, chomp: true)).freeze
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# IANA's own version string for the bundled snapshot, e.g.
|
|
77
|
+
# "2026082301, Last Updated Mon Aug 24 07:07:01 2026 UTC". Useful in a
|
|
78
|
+
# health check to see how old your list is.
|
|
79
|
+
def version
|
|
80
|
+
@version ||= begin
|
|
81
|
+
header = File.open(LIST_PATH, &:readline).to_s.strip
|
|
82
|
+
header.start_with?('#') ? header.sub(/\A#\s*Version\s*/i, '') : nil
|
|
83
|
+
rescue StandardError
|
|
84
|
+
nil
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# The TLD of an email address or a bare domain, downcased.
|
|
89
|
+
#
|
|
90
|
+
# nil means THERE IS NO TLD HERE — which is not the same as "we can't
|
|
91
|
+
# tell". `example@gmailmcom` (the dot missed entirely) and
|
|
92
|
+
# `you@localhost` both land here, and under check_tld both are rejected:
|
|
93
|
+
# a domain with no TLD cannot end in a real one. Ask #judgeable? first if
|
|
94
|
+
# you need to tell that apart from an address this gem has no opinion on.
|
|
95
|
+
def extract(email_or_domain)
|
|
96
|
+
domain = normalize(email_or_domain)
|
|
97
|
+
return nil if domain.empty?
|
|
98
|
+
|
|
99
|
+
labels = domain.split('.')
|
|
100
|
+
return nil if labels.size < 2
|
|
101
|
+
|
|
102
|
+
tld = labels.last
|
|
103
|
+
tld.empty? ? nil : tld
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# Is this an address the TLD rules have anything to say about?
|
|
107
|
+
#
|
|
108
|
+
# False for the two shapes where "what is the TLD" is the wrong question
|
|
109
|
+
# rather than a question with a bad answer: an empty domain, and an IP
|
|
110
|
+
# literal (`user@192.168.0.1`, `user@[10.0.0.1]`). Both are left alone —
|
|
111
|
+
# deciding whether to accept an IP-literal address is a format policy, and
|
|
112
|
+
# a validator called `check_tld` has no business making it.
|
|
113
|
+
def judgeable?(email_or_domain)
|
|
114
|
+
domain = normalize(email_or_domain)
|
|
115
|
+
return false if domain.empty?
|
|
116
|
+
return false if domain.start_with?('[') || domain.match?(/\A[\d.]+\z/)
|
|
117
|
+
|
|
118
|
+
true
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# Is this TLD in the root zone (or in the host's additional_tlds)?
|
|
122
|
+
#
|
|
123
|
+
# ⚠️ Non-ASCII TLDs always answer true. IANA lists internationalised TLDs
|
|
124
|
+
# in punycode (`XN--FIQS8S`), and converting `中国` to that form needs an
|
|
125
|
+
# IDN library we deliberately do not depend on. Rather than reject every
|
|
126
|
+
# unicode address, we decline to judge them: a false accept is a nuisance,
|
|
127
|
+
# a false reject is a locked-out human. Punycode-form addresses, which is
|
|
128
|
+
# what mail clients actually send, are checked normally.
|
|
129
|
+
def valid?(tld)
|
|
130
|
+
tld = tld.to_s.downcase
|
|
131
|
+
return false if tld.empty?
|
|
132
|
+
return true unless tld.ascii_only?
|
|
133
|
+
|
|
134
|
+
all.include?(tld) || configured(:additional_tlds).include?(tld)
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# Has the host explicitly blocked this TLD — either by naming it in
|
|
138
|
+
# `blocked_tlds`, or by naming everything else in `allowed_tlds`?
|
|
139
|
+
def blocked?(tld)
|
|
140
|
+
tld = tld.to_s.downcase
|
|
141
|
+
return false if tld.empty?
|
|
142
|
+
|
|
143
|
+
allowed = configured(:allowed_tlds)
|
|
144
|
+
return true if allowed && !allowed.include?(tld)
|
|
145
|
+
|
|
146
|
+
configured(:blocked_tlds).include?(tld)
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# Reset the memoized list. Called by the updater after it rewrites the
|
|
150
|
+
# file, and by tests.
|
|
151
|
+
def reload!
|
|
152
|
+
@all = nil
|
|
153
|
+
@version = nil
|
|
154
|
+
self
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
private
|
|
158
|
+
|
|
159
|
+
# The domain half in Nondisposable::Domain's canonical form, minus the
|
|
160
|
+
# punycode step: trimmed, downcased, no trailing root dot, fullwidth and
|
|
161
|
+
# ideographic dots read as dots (`foo.tk` ends in `tk`, exactly like
|
|
162
|
+
# `foo.tk`). Unicode TLDs stay unicode, so #valid? still declines to
|
|
163
|
+
# judge them.
|
|
164
|
+
def normalize(email_or_domain)
|
|
165
|
+
value = email_or_domain.to_s
|
|
166
|
+
value = value.b unless value.valid_encoding? # unreadable bytes must not raise
|
|
167
|
+
Nondisposable::Domain.unicode(value.split('@').last.to_s)
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
# Skips IANA's comment header and any blank lines.
|
|
171
|
+
def parse(lines)
|
|
172
|
+
lines.each_with_object(Set.new) do |line, set|
|
|
173
|
+
value = line.to_s.strip.downcase
|
|
174
|
+
next if value.empty? || value.start_with?('#')
|
|
175
|
+
|
|
176
|
+
set << value
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
# Config lists are host-supplied, so normalise on every read rather than
|
|
181
|
+
# trusting them to be lowercase, dotless strings. `.tk` and `TK` both work.
|
|
182
|
+
def configured(key)
|
|
183
|
+
value = Nondisposable.configuration.public_send(key)
|
|
184
|
+
return nil if value.nil?
|
|
185
|
+
|
|
186
|
+
Set.new(Array(value).map { |tld| tld.to_s.strip.downcase.delete_prefix('.') })
|
|
187
|
+
end
|
|
188
|
+
end
|
|
189
|
+
end
|
|
190
|
+
end
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'net/http'
|
|
4
|
+
require 'uri'
|
|
5
|
+
|
|
6
|
+
module Nondisposable
|
|
7
|
+
# Refreshes the vendored IANA TLD snapshot from the root zone database.
|
|
8
|
+
#
|
|
9
|
+
# ⚠️ This is a MAINTENANCE task, not a runtime one — the deliberate opposite of
|
|
10
|
+
# DomainListUpdater. That one runs in your app, on a schedule, because
|
|
11
|
+
# disposable domains change every few days and live in your database. This one
|
|
12
|
+
# rewrites a file inside the gem, which is read-only once the gem is installed.
|
|
13
|
+
# Run it in a checkout of the gem before cutting a release:
|
|
14
|
+
#
|
|
15
|
+
# rake nondisposable:tlds:update
|
|
16
|
+
#
|
|
17
|
+
# If you need a TLD that is newer than your installed gem, do NOT reach for
|
|
18
|
+
# this. Add it to `config.additional_tlds` and you are unblocked immediately,
|
|
19
|
+
# with no release and no writable-gem-directory problem:
|
|
20
|
+
#
|
|
21
|
+
# config.additional_tlds = %w[newtld]
|
|
22
|
+
#
|
|
23
|
+
# New TLDs are delegated in batches a few times a year, so a snapshot ages
|
|
24
|
+
# slowly. ICANN's next application round is the first real test of that.
|
|
25
|
+
class TldListUpdater
|
|
26
|
+
LIST_URL = 'https://data.iana.org/TLD/tlds-alpha-by-domain.txt'
|
|
27
|
+
|
|
28
|
+
OPEN_TIMEOUT = 10
|
|
29
|
+
READ_TIMEOUT = 10
|
|
30
|
+
|
|
31
|
+
# A root zone that suddenly contains a handful of entries means the fetch
|
|
32
|
+
# went wrong (a captive portal, an error page, a truncated body), not that
|
|
33
|
+
# ICANN deleted the internet. Refuse to overwrite a good list with junk.
|
|
34
|
+
MINIMUM_PLAUSIBLE_TLD_COUNT = 1_000
|
|
35
|
+
|
|
36
|
+
class << self
|
|
37
|
+
# Returns the number of TLDs written, or nil if nothing was written.
|
|
38
|
+
def update(path: Nondisposable::Tld::LIST_PATH)
|
|
39
|
+
log :info, "Fetching the IANA root zone TLD list from #{LIST_URL}..."
|
|
40
|
+
|
|
41
|
+
response = fetch
|
|
42
|
+
unless response.is_a?(Net::HTTPSuccess)
|
|
43
|
+
log :error, "Failed to download the TLD list. HTTP status: #{response.code}. Keeping the existing snapshot."
|
|
44
|
+
return nil
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
body = response.body.to_s
|
|
48
|
+
count = body.lines.count { |line| meaningful?(line) }
|
|
49
|
+
|
|
50
|
+
if count < MINIMUM_PLAUSIBLE_TLD_COUNT
|
|
51
|
+
log :error, "Refusing to write a TLD list with only #{count} entries (expected at least #{MINIMUM_PLAUSIBLE_TLD_COUNT}). Keeping the existing snapshot."
|
|
52
|
+
return nil
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Written whole, then moved into place, so an interrupted run can never
|
|
56
|
+
# leave a half-written root zone behind.
|
|
57
|
+
tmp = "#{path}.tmp"
|
|
58
|
+
File.write(tmp, body)
|
|
59
|
+
File.rename(tmp, path)
|
|
60
|
+
Nondisposable::Tld.reload!
|
|
61
|
+
|
|
62
|
+
log :info, "Wrote #{count} TLDs to #{path} (#{Nondisposable::Tld.version})."
|
|
63
|
+
count
|
|
64
|
+
rescue StandardError => e
|
|
65
|
+
log :error, "Could not update the TLD list: #{e.class}: #{e.message}. Keeping the existing snapshot."
|
|
66
|
+
nil
|
|
67
|
+
ensure
|
|
68
|
+
File.delete(tmp) if tmp && File.exist?(tmp)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
private
|
|
72
|
+
|
|
73
|
+
def meaningful?(line)
|
|
74
|
+
value = line.to_s.strip
|
|
75
|
+
!value.empty? && !value.start_with?('#')
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def fetch
|
|
79
|
+
uri = URI(LIST_URL)
|
|
80
|
+
Net::HTTP.start(
|
|
81
|
+
uri.host,
|
|
82
|
+
uri.port,
|
|
83
|
+
use_ssl: uri.scheme == 'https',
|
|
84
|
+
open_timeout: OPEN_TIMEOUT,
|
|
85
|
+
read_timeout: READ_TIMEOUT
|
|
86
|
+
) { |http| http.get(uri.request_uri) }
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# Usable from a bare `rake` run in a gem checkout, where there is no Rails
|
|
90
|
+
# logger to talk to.
|
|
91
|
+
def log(level, message)
|
|
92
|
+
message = "[nondisposable] #{message}"
|
|
93
|
+
if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
|
|
94
|
+
Rails.logger.public_send(level, message)
|
|
95
|
+
else
|
|
96
|
+
warn message
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
end
|