nondisposable 0.3.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +73 -0
- data/README.md +305 -3
- data/Rakefile +5 -0
- data/app/models/nondisposable/disposable_domain.rb +53 -34
- data/app/models/nondisposable/source_snapshot.rb +66 -0
- data/data/email_providers.txt +276 -0
- data/data/iana_tlds.txt +1439 -0
- data/data/known_typo_domains.txt +99 -0
- data/data/never_disposable.txt +480 -0
- data/data/privacy_relays.txt +42 -0
- data/data/public_suffix_list.dat +16571 -0
- data/gemfiles/rails_7.2.gemfile +1 -0
- data/gemfiles/rails_8.0.gemfile +1 -0
- data/gemfiles/rails_8.1.gemfile +1 -0
- data/lib/generators/nondisposable/install_generator.rb +1 -0
- data/lib/generators/nondisposable/templates/create_nondisposable_source_snapshots.rb.erb +42 -0
- data/lib/generators/nondisposable/templates/nondisposable.rb +67 -1
- data/lib/generators/nondisposable/upgrade_generator.rb +43 -0
- data/lib/nondisposable/confirmable_email.rb +75 -0
- data/lib/nondisposable/domain.rb +238 -0
- data/lib/nondisposable/domain_list_updater.rb +510 -88
- data/lib/nondisposable/email_validator.rb +101 -3
- data/lib/nondisposable/known_typos.rb +64 -0
- data/lib/nondisposable/precedence.rb +209 -0
- data/lib/nondisposable/public_suffix.rb +147 -0
- data/lib/nondisposable/public_suffix_list_updater.rb +88 -0
- data/lib/nondisposable/source.rb +233 -0
- data/lib/nondisposable/source_fetcher.rb +69 -0
- data/lib/nondisposable/source_merge.rb +127 -0
- data/lib/nondisposable/suggestion.rb +336 -0
- data/lib/nondisposable/tld.rb +190 -0
- data/lib/nondisposable/tld_list_updater.rb +101 -0
- data/lib/nondisposable/version.rb +1 -1
- data/lib/nondisposable.rb +153 -6
- data/lib/tasks/nondisposable.rake +66 -0
- metadata +24 -2
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Nondisposable
|
|
4
|
+
# What one source said on one fetch — or why we refused to listen.
|
|
5
|
+
#
|
|
6
|
+
# The merged nondisposable_disposable_domains table cannot answer "which
|
|
7
|
+
# list put this domain here?", and it cannot be rebuilt when one source has a
|
|
8
|
+
# bad day: a single domain may come from several sources, and a domain one
|
|
9
|
+
# corroborating source lists must survive until a second one agrees. So each
|
|
10
|
+
# source keeps its own recent history here, and the merged table is always
|
|
11
|
+
# REBUILT from each source's last good snapshot (DomainListUpdater.publish).
|
|
12
|
+
#
|
|
13
|
+
# Statuses:
|
|
14
|
+
# ok accepted; `domains` holds the canonical list, one per line
|
|
15
|
+
# rejected downloaded, but failed a guard (too small, shrank or grew
|
|
16
|
+
# implausibly, mostly junk); `error` says which
|
|
17
|
+
# failed the download itself failed; `error` says how
|
|
18
|
+
# rolled_back an ok snapshot a human withdrew (DomainListUpdater.rollback)
|
|
19
|
+
#
|
|
20
|
+
# Only `ok` snapshots ever feed the published list. The table is optional:
|
|
21
|
+
# without it (hosts that have not run `rails g nondisposable:upgrade`), the
|
|
22
|
+
# updater keeps the single-source behaviour of 0.5 and earlier.
|
|
23
|
+
class SourceSnapshot < ApplicationRecord
|
|
24
|
+
self.table_name = 'nondisposable_source_snapshots'
|
|
25
|
+
|
|
26
|
+
STATUSES = %w[ok rejected failed rolled_back].freeze
|
|
27
|
+
|
|
28
|
+
# Rows kept per source: the newest RETAINED_PER_SOURCE of any status, plus
|
|
29
|
+
# the newest RETAINED_OK_PER_SOURCE good ones however old (failures must
|
|
30
|
+
# never evict the fallback, nor the shrink guard's baseline), plus EVERY
|
|
31
|
+
# rolled-back row (its sha256 is what refuses the same poison next time).
|
|
32
|
+
RETAINED_PER_SOURCE = 5
|
|
33
|
+
RETAINED_OK_PER_SOURCE = 3
|
|
34
|
+
|
|
35
|
+
validates :source, :status, :fetched_at, presence: true
|
|
36
|
+
validates :status, inclusion: { in: STATUSES }
|
|
37
|
+
|
|
38
|
+
scope :ok, -> { where(status: 'ok') }
|
|
39
|
+
scope :newest_first, -> { order(fetched_at: :desc, id: :desc) }
|
|
40
|
+
|
|
41
|
+
class << self
|
|
42
|
+
# Is the table there? Checked on every update rather than memoized, so a
|
|
43
|
+
# long-running process notices the upgrade migration without a restart.
|
|
44
|
+
def available?
|
|
45
|
+
connection_pool.with_connection { |connection| connection.data_source_exists?(table_name) }
|
|
46
|
+
rescue StandardError
|
|
47
|
+
false
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def last_good(source_name)
|
|
51
|
+
ok.where(source: source_name).newest_first.first
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def prune(source_name)
|
|
55
|
+
rows = where(source: source_name)
|
|
56
|
+
keep = rows.newest_first.limit(RETAINED_PER_SOURCE).pluck(:id) +
|
|
57
|
+
rows.ok.newest_first.limit(RETAINED_OK_PER_SOURCE).pluck(:id)
|
|
58
|
+
rows.where.not(id: keep).where.not(status: 'rolled_back').delete_all
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def domain_list
|
|
63
|
+
domains.to_s.split("\n")
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
# Well-known consumer email domains, used by Nondisposable::Suggestion.
|
|
2
|
+
#
|
|
3
|
+
# This list does TWO jobs, and the second one is the reason it needs to be
|
|
4
|
+
# generous rather than short:
|
|
5
|
+
#
|
|
6
|
+
# 1. It supplies the candidates a typo can be corrected TO. A domain one edit
|
|
7
|
+
# away from an entry here is very probably a slip of the finger.
|
|
8
|
+
# 2. It is the false-positive guard. An EXACT match here is never flagged, so
|
|
9
|
+
# every real provider that happens to sit one edit from another real one
|
|
10
|
+
# must be present. `mail.com` is the canonical example: it is one insertion
|
|
11
|
+
# away from `gmail.com`, and leaving it out would tell every one of its
|
|
12
|
+
# users they had made a typo.
|
|
13
|
+
#
|
|
14
|
+
# Hosts add their own with `config.additional_email_providers`. Order is
|
|
15
|
+
# irrelevant; comments and blank lines are ignored.
|
|
16
|
+
|
|
17
|
+
# --- global ---
|
|
18
|
+
gmail.com
|
|
19
|
+
googlemail.com
|
|
20
|
+
outlook.com
|
|
21
|
+
hotmail.com
|
|
22
|
+
live.com
|
|
23
|
+
msn.com
|
|
24
|
+
passport.com
|
|
25
|
+
yahoo.com
|
|
26
|
+
ymail.com
|
|
27
|
+
rocketmail.com
|
|
28
|
+
icloud.com
|
|
29
|
+
me.com
|
|
30
|
+
mac.com
|
|
31
|
+
aol.com
|
|
32
|
+
mail.com
|
|
33
|
+
email.com
|
|
34
|
+
usa.com
|
|
35
|
+
gmx.com
|
|
36
|
+
gmx.net
|
|
37
|
+
gmx.de
|
|
38
|
+
gmx.at
|
|
39
|
+
gmx.ch
|
|
40
|
+
zoho.com
|
|
41
|
+
zohomail.com
|
|
42
|
+
fastmail.com
|
|
43
|
+
fastmail.fm
|
|
44
|
+
hushmail.com
|
|
45
|
+
protonmail.com
|
|
46
|
+
protonmail.ch
|
|
47
|
+
proton.me
|
|
48
|
+
pm.me
|
|
49
|
+
tutanota.com
|
|
50
|
+
tutanota.de
|
|
51
|
+
tuta.com
|
|
52
|
+
tuta.io
|
|
53
|
+
hey.com
|
|
54
|
+
mailfence.com
|
|
55
|
+
posteo.de
|
|
56
|
+
disroot.org
|
|
57
|
+
riseup.net
|
|
58
|
+
runbox.com
|
|
59
|
+
migadu.com
|
|
60
|
+
purelymail.com
|
|
61
|
+
skiff.com
|
|
62
|
+
duck.com
|
|
63
|
+
|
|
64
|
+
# --- Spain ---
|
|
65
|
+
hotmail.es
|
|
66
|
+
outlook.es
|
|
67
|
+
yahoo.es
|
|
68
|
+
live.com.es
|
|
69
|
+
telefonica.net
|
|
70
|
+
movistar.es
|
|
71
|
+
terra.es
|
|
72
|
+
terra.com
|
|
73
|
+
ya.com
|
|
74
|
+
wanadoo.es
|
|
75
|
+
orange.es
|
|
76
|
+
jazzfree.com
|
|
77
|
+
jazztel.es
|
|
78
|
+
euskaltel.net
|
|
79
|
+
euskalnet.net
|
|
80
|
+
ono.com
|
|
81
|
+
vodafone.es
|
|
82
|
+
telecable.es
|
|
83
|
+
mundo-r.com
|
|
84
|
+
airtel.net
|
|
85
|
+
navegalia.com
|
|
86
|
+
eresmas.com
|
|
87
|
+
inicia.es
|
|
88
|
+
teleline.es
|
|
89
|
+
|
|
90
|
+
# --- Portugal / Latin America ---
|
|
91
|
+
sapo.pt
|
|
92
|
+
netcabo.pt
|
|
93
|
+
clix.pt
|
|
94
|
+
uol.com.br
|
|
95
|
+
bol.com.br
|
|
96
|
+
ig.com.br
|
|
97
|
+
terra.com.br
|
|
98
|
+
globo.com
|
|
99
|
+
globomail.com
|
|
100
|
+
prodigy.net.mx
|
|
101
|
+
hotmail.com.ar
|
|
102
|
+
hotmail.com.mx
|
|
103
|
+
yahoo.com.ar
|
|
104
|
+
yahoo.com.mx
|
|
105
|
+
yahoo.com.br
|
|
106
|
+
|
|
107
|
+
# --- France ---
|
|
108
|
+
orange.fr
|
|
109
|
+
wanadoo.fr
|
|
110
|
+
free.fr
|
|
111
|
+
sfr.fr
|
|
112
|
+
laposte.net
|
|
113
|
+
bbox.fr
|
|
114
|
+
neuf.fr
|
|
115
|
+
aliceadsl.fr
|
|
116
|
+
club-internet.fr
|
|
117
|
+
numericable.fr
|
|
118
|
+
hotmail.fr
|
|
119
|
+
outlook.fr
|
|
120
|
+
yahoo.fr
|
|
121
|
+
live.fr
|
|
122
|
+
|
|
123
|
+
# --- Germany / Austria / Switzerland ---
|
|
124
|
+
web.de
|
|
125
|
+
t-online.de
|
|
126
|
+
freenet.de
|
|
127
|
+
arcor.de
|
|
128
|
+
1und1.de
|
|
129
|
+
online.de
|
|
130
|
+
gmx.info
|
|
131
|
+
hotmail.de
|
|
132
|
+
outlook.de
|
|
133
|
+
yahoo.de
|
|
134
|
+
live.de
|
|
135
|
+
bluewin.ch
|
|
136
|
+
sunrise.ch
|
|
137
|
+
gmail.ch
|
|
138
|
+
|
|
139
|
+
# --- United Kingdom / Ireland ---
|
|
140
|
+
btinternet.com
|
|
141
|
+
sky.com
|
|
142
|
+
virginmedia.com
|
|
143
|
+
talktalk.net
|
|
144
|
+
ntlworld.com
|
|
145
|
+
blueyonder.co.uk
|
|
146
|
+
hotmail.co.uk
|
|
147
|
+
outlook.co.uk
|
|
148
|
+
yahoo.co.uk
|
|
149
|
+
live.co.uk
|
|
150
|
+
googlemail.co.uk
|
|
151
|
+
eircom.net
|
|
152
|
+
|
|
153
|
+
# --- Italy ---
|
|
154
|
+
libero.it
|
|
155
|
+
virgilio.it
|
|
156
|
+
alice.it
|
|
157
|
+
tiscali.it
|
|
158
|
+
tin.it
|
|
159
|
+
fastwebnet.it
|
|
160
|
+
hotmail.it
|
|
161
|
+
outlook.it
|
|
162
|
+
yahoo.it
|
|
163
|
+
live.it
|
|
164
|
+
pec.it
|
|
165
|
+
|
|
166
|
+
# --- Benelux / Nordics / other Europe ---
|
|
167
|
+
telenet.be
|
|
168
|
+
skynet.be
|
|
169
|
+
proximus.be
|
|
170
|
+
ziggo.nl
|
|
171
|
+
kpnmail.nl
|
|
172
|
+
home.nl
|
|
173
|
+
planet.nl
|
|
174
|
+
xs4all.nl
|
|
175
|
+
telia.com
|
|
176
|
+
telia.se
|
|
177
|
+
bredband.net
|
|
178
|
+
spray.se
|
|
179
|
+
hotmail.se
|
|
180
|
+
online.no
|
|
181
|
+
broadpark.no
|
|
182
|
+
tele2.no
|
|
183
|
+
elisanet.fi
|
|
184
|
+
luukku.com
|
|
185
|
+
suomi24.fi
|
|
186
|
+
seznam.cz
|
|
187
|
+
centrum.cz
|
|
188
|
+
wp.pl
|
|
189
|
+
o2.pl
|
|
190
|
+
onet.pl
|
|
191
|
+
interia.pl
|
|
192
|
+
gazeta.pl
|
|
193
|
+
abv.bg
|
|
194
|
+
mail.bg
|
|
195
|
+
freemail.hu
|
|
196
|
+
citromail.hu
|
|
197
|
+
t-online.hu
|
|
198
|
+
|
|
199
|
+
# --- Russia / Eastern Europe ---
|
|
200
|
+
mail.ru
|
|
201
|
+
inbox.ru
|
|
202
|
+
list.ru
|
|
203
|
+
bk.ru
|
|
204
|
+
internet.ru
|
|
205
|
+
yandex.ru
|
|
206
|
+
yandex.com
|
|
207
|
+
ya.ru
|
|
208
|
+
rambler.ru
|
|
209
|
+
ukr.net
|
|
210
|
+
i.ua
|
|
211
|
+
meta.ua
|
|
212
|
+
|
|
213
|
+
# --- Asia ---
|
|
214
|
+
qq.com
|
|
215
|
+
foxmail.com
|
|
216
|
+
163.com
|
|
217
|
+
126.com
|
|
218
|
+
yeah.net
|
|
219
|
+
sina.com
|
|
220
|
+
sina.cn
|
|
221
|
+
sohu.com
|
|
222
|
+
aliyun.com
|
|
223
|
+
naver.com
|
|
224
|
+
daum.net
|
|
225
|
+
hanmail.net
|
|
226
|
+
nate.com
|
|
227
|
+
kakao.com
|
|
228
|
+
docomo.ne.jp
|
|
229
|
+
ezweb.ne.jp
|
|
230
|
+
softbank.ne.jp
|
|
231
|
+
yahoo.co.jp
|
|
232
|
+
ybb.ne.jp
|
|
233
|
+
nifty.com
|
|
234
|
+
biglobe.ne.jp
|
|
235
|
+
so-net.ne.jp
|
|
236
|
+
rediffmail.com
|
|
237
|
+
sify.com
|
|
238
|
+
|
|
239
|
+
# --- North America ISPs ---
|
|
240
|
+
comcast.net
|
|
241
|
+
xfinity.com
|
|
242
|
+
verizon.net
|
|
243
|
+
att.net
|
|
244
|
+
sbcglobal.net
|
|
245
|
+
bellsouth.net
|
|
246
|
+
ameritech.net
|
|
247
|
+
pacbell.net
|
|
248
|
+
cox.net
|
|
249
|
+
charter.net
|
|
250
|
+
spectrum.net
|
|
251
|
+
earthlink.net
|
|
252
|
+
juno.com
|
|
253
|
+
netzero.net
|
|
254
|
+
optonline.net
|
|
255
|
+
roadrunner.com
|
|
256
|
+
rr.com
|
|
257
|
+
windstream.net
|
|
258
|
+
frontier.com
|
|
259
|
+
centurylink.net
|
|
260
|
+
shaw.ca
|
|
261
|
+
rogers.com
|
|
262
|
+
sympatico.ca
|
|
263
|
+
telus.net
|
|
264
|
+
videotron.ca
|
|
265
|
+
bell.net
|
|
266
|
+
|
|
267
|
+
# --- Oceania / Africa ---
|
|
268
|
+
bigpond.com
|
|
269
|
+
bigpond.net.au
|
|
270
|
+
optusnet.com.au
|
|
271
|
+
iinet.net.au
|
|
272
|
+
tpg.com.au
|
|
273
|
+
xtra.co.nz
|
|
274
|
+
webmail.co.za
|
|
275
|
+
telkomsa.net
|
|
276
|
+
mweb.co.za
|