RubyGems - email_data - Versions diffs - 1604944785 → 1605483437 - Mend

email_data 1604944785 → 1605483437

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (17) hide show

checksums.yaml +4 -4
data/Gemfile.lock +5 -4
data/README.md +34 -0
data/VERSION +1 -1
data/bin/publish-npm +0 -6
data/bin/sync +2 -0
data/bin/sync-disposable-emails +6 -3
data/bin/sync-tld +15 -0
data/data/disposable_domains.txt +47 -0
data/data/manual/private_relays.txt +1 -0
data/data/private_relays.txt +1 -0
data/data/slds.txt +5831 -0
data/lib/email_data.rb +2 -0
data/lib/email_data/source/active_record.rb +16 -0
data/lib/email_data/source/file_system.rb +8 -0
data/package.json +1 -1
metadata +5 -2

checksums.yaml CHANGED

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aa8d3696d76e382a9c5f4dafd80239dae34fa4f5a16ba17108ebe7c65e3092cb
-  data.tar.gz: 8d8a9f1228ca1fba0bc4931df8710ae11bb92e85995b46b438c137024fd8101a
+  metadata.gz: 7ee4f3cd6480023a786dddb8575eef1e905c483ae039fc28fd83a58801a10898
+  data.tar.gz: 00c365370cd541acbac7815975e7fb2bba3642b1a2fe67c9babd423fbbad2aa8
 SHA512:
-  metadata.gz: d2c9c9e455690c0d6e9fff0d9caced96e7207e7aae668b977acbbc8efe054d0a968351cdcaafd043172e5f0dac46ace5ca74df757135ff3a7336f4c93ac3c3ec
-  data.tar.gz: 24b3f3346e4b2a602b503038ed2e97ab232c3cd47265a4391798b113d2c4ced176b3d4fac33016c5e145fb22c7c7e53dd0cd94df897d839139233123a302988b
+  metadata.gz: ab4ee9e948f586672c4a6e1a54ef1923853ca29493e9926f61d4eb6c2b1c40e35e660957d28c74d99c7e0dc3b773f34c9bd4bb0267bfb3380fe82a6750f30434
+  data.tar.gz: 283ac7b4918b5fc74fb5edaca8a59af3ad0da8173eea5815b853fb2e46c9b2747585d5a46338e9d246a022c0ec40f446ea1450e0945fbb489d024ddd22ebf667

data/Gemfile.lock CHANGED

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    email_data (1604944785)
+    email_data (1605483437)
 GEM
   remote: https://rubygems.org/
@@ -38,15 +38,16 @@ GEM
     rake (13.0.1)
     regexp_parser (1.8.2)
     rexml (3.2.4)
-    root_domain (0.1.0)
+    root_domain (0.1.1)
+      email_data
       simpleidn
-    rubocop (1.2.0)
+    rubocop (1.3.0)
       parallel (~> 1.10)
       parser (>= 2.7.1.5)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8)
       rexml
-      rubocop-ast (>= 1.0.1)
+      rubocop-ast (>= 1.1.1)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 2.0)
     rubocop-ast (1.1.1)

data/README.md CHANGED

@@ -58,6 +58,18 @@ EmailData.free_email_domains
 # List of roles. Can be used to filter out emails like info@ or all@.
 EmailData.roles
+# List of private relays like Apple's Hide My Email.
+EmailData.private_relays
+# List of country tlds.
+EmailData.country_tlds
+# List of official tlds.
+EmailData.tlds
+# List of second-level domains.
+EmailData.slds
 ```
 #### Data sources
@@ -98,6 +110,12 @@ class SetupEmailData < ActiveRecord::Migration[6.1]
     add_index :tlds, :name, unique: true
+    create_table :slds do |t|
+      t.citext :name, null: false
+    end
+    add_index :slds, :name, unique: true
     create_table :country_tlds do |t|
       t.citext :name, null: false
     end
@@ -127,6 +145,12 @@ class SetupEmailData < ActiveRecord::Migration[6.1]
     end
     add_index :roles, :name, unique: true
+    create_table :private_relays do |t|
+      t.citext :name, null: false
+    end
+    add_index :private_relays, :name, unique: true
   end
 end
 ```
@@ -155,11 +179,13 @@ The you can load each dataset using `COPY`:
 ```sql
 COPY tlds (name) FROM '/usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0/gems/email_data-1601479967/data/tlds.txt';
+COPY slds (name) FROM '/usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0/gems/email_data-1601479967/data/slds.txt';
 COPY country_tlds (name) FROM '/usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0/gems/email_data-1601479967/data/country_tlds.txt';
 COPY disposable_emails (name) FROM '/usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0/gems/email_data-1601479967/data/disposable_emails.txt';
 COPY disposable_domains (name) FROM '/usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0/gems/email_data-1601479967/data/disposable_domains.txt';
 COPY free_email_domains (name) FROM '/usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0/gems/email_data-1601479967/data/free_email_domains.txt';
 COPY roles (name) FROM '/usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0/gems/email_data-1601479967/data/roles.txt';
+COPY private_relays (name) FROM '/usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0/gems/email_data-1601479967/data/private_relays.txt';
 ```
 Alternatively, you could create a migrate that executes that same command; given
@@ -181,11 +207,13 @@ class LoadEmailData < ActiveRecord::Migration[6.1]
     end
     copy.call(:tlds)
+    copy.call(:slds)
     copy.call(:country_tlds)
     copy.call(:disposable_emails)
     copy.call(:disposable_domains)
     copy.call(:free_email_domains)
     copy.call(:roles)
+    copy.call(:private_relays)
   end
 end
 ```
@@ -209,6 +237,10 @@ const disposableEmails = require("@fnando/email_data/data/json/disposable_emails
 const disposableDomains = require("@fnando/email_data/data/json/disposable_domains.json");
 const freeEmailDomains = require("@fnando/email_data/data/json/free_email_domains.json");
 const roles = require("@fnando/email_data/data/json/roles.json");
+const privateRelays = require("@fnando/email_data/data/json/private_relays.json");
+const tlds = require("@fnando/email_data/data/json/tlds.json");
+const slds = require("@fnando/email_data/data/json/slds.json");
+const cctlds = require("@fnando/email_data/data/json/country_tlds.json");
 ```
 ## Dataset
@@ -224,6 +256,8 @@ like to add, please make a pull request against the files `data/manual/*.txt`.
 - `data/manual/free_email_domains.txt`: only free email services must go here.
 - `data/manual/roles.txt`: list of role-based user names like `info` or
   `no-reply`.
+- `data/manual/private_relays.txt`: list of private relay services, like Apple's
+  Hide My Email.
 ## Development

data/VERSION CHANGED

	@@ -1 +1 @@
1	- ~~1604944785~~
1	+ 1605483437

data/bin/publish-npm CHANGED

@@ -7,12 +7,6 @@ source $HOME/.zsh/asdf.sh
 asdf local nodejs 12.18.4
-for input in $(ls -1 data/*.txt); do
-  name=$(basename $input)
-  output=data/json/"${name%.txt}.json"
-  jq -R -s -c 'split("\n")' < $input > $output
-done
 npm publish --access=public
 terminal-notifier \

data/bin/sync CHANGED

@@ -5,3 +5,5 @@ echo
 ./bin/sync-free-emails
 echo
 ./bin/sync-disposable-emails
+echo
+./bin/export-json-files

data/bin/sync-disposable-emails CHANGED

@@ -182,6 +182,10 @@ threads.clear
 domains = []
+puts "=> Saving private relays"
+private_relays = normalize_list(File.read("#{__dir__}/../data/manual/private_relays.txt").lines)
+save_file("private_relays.txt", private_relays)
 puts "=> Loading disposable_domains.txt"
 domains += normalize_list(File.read("#{__dir__}/../data/disposable_domains.txt").lines)
@@ -196,9 +200,8 @@ end
 ignore_domains = normalize_list(File.read("#{__dir__}/../data/free_email_domains.txt").lines)
                  .map {|domain| RootDomain.call(domain) }
-ignore_domains += %w[
-  appleid.com
-]
+ignore_domains += normalize_list(File.read("#{__dir__}/../data/private_relays.txt").lines)
+                  .map {|line| RootDomain.call(line) }
 puts "=> Normalize domains (count: #{domains.size})"
 domains = domains

data/bin/sync-tld CHANGED

@@ -26,3 +26,18 @@ country_tlds = country_tlds
 puts "=> Saving country_tlds.txt"
 save_file("country_tlds.txt", normalize_list(country_tlds))
+puts "=> Fetching sld list"
+response = http_request(:get, "https://publicsuffix.org/list/public_suffix_list.dat")
+content, _ = response.body.split("END ICANN DOMAINS")
+slds = content
+       .lines
+       .map {|line| line.gsub(/^\*\./, "") }
+       .map(&:strip)
+       .reject(&:empty?)
+       .reject {|line| line.start_with?("/") }
+       .map {|line| line.gsub(/^([!*.]+)/, "") }
+       .reject {|line| line.split(".").size == 1 }
+save_file("slds.txt", normalize_list(slds))

data/data/disposable_domains.txt CHANGED

@@ -2215,6 +2215,7 @@
 25806.monster
 2584445.com
 2594445.com
+2595966.com
 25betturka.com
 25comp.ru
 25fwlp1ih7av5ym.xyz
@@ -7266,6 +7267,7 @@
 97178c5.com
 97178c6.com
 97207q.com
+9722.us
 9724445.com
 9727a.com
 973.dog
@@ -10356,6 +10358,7 @@ alhadattv.com
 alhalfpricelistings.com
 alhamadealmeria.com
 alhashareslectronics.com
+alhemyaria.news
 alhzfw.us
 ali-baba.info
 aliagaberrakemlak.com
@@ -10429,6 +10432,7 @@ alisverisistan.net
 alitaj.com
 alittle.website
 alivance.com
+aliveinlosangeles.com
 alivemail.cf
 alivemail.ga
 alivemail.gq
@@ -12000,6 +12004,7 @@ apk-download.xyz
 apk1000.com
 apk2download.net
 apkfun.club
+apklitestore.com
 apkload.com
 apkmd.com
 apkmoe.com
@@ -13484,6 +13489,7 @@ aub6jbkk.xyz
 aubootfans.co.uk
 aubootfans.com
 aubootsoutlet.co.uk
+aubreyequine.com
 aucenter.info
 auchandirekt.pl
 auction315.xyz
@@ -13578,6 +13584,7 @@ ausdoctors.info
 ausgefallen.info
 auslaenderberatunge-erfurt.press
 auspaccornerstone.com
+auspb.com
 auspicy.best
 aussie.finance
 aussie.loan
@@ -14289,6 +14296,7 @@ azspartners.ru
 azsportsnetwork.net
 aztecspecialties.us
 azterul.ru
+aztraumacenter.com
 azulaomarine.com
 azulgold.com
 azumail.com
@@ -14442,6 +14450,7 @@ b55b56.gq
 b55b56.ml
 b55b56.tk
 b571qqnl.buzz
+b572elzs.buzz
 b59da.xyz
 b59qph.us
 b5gtfghtt.xyz
@@ -16513,6 +16522,7 @@ bestcustomlogo.com
 bestdarkspotcorrector.org
 bestdates.xyz
 bestdateshere23.com
+bestdavenportchiropractor.com
 bestday.pw
 bestdealsdiscounts.co.in
 bestdentistofficenearme.com
@@ -18130,6 +18140,7 @@ bjog.tech
 bjq53.space
 bjqjvtodz.shop
 bjsnc.com
+bjsy6p.com
 bjtpg.us
 bjwc.us
 bjxt.ml
@@ -20770,6 +20781,7 @@ buylaptopsunder300.com
 buylevitra-us.com
 buylevitra.website
 buylikes247.com
+buylimu.biz
 buylouisvuittonbagsjp.com
 buymethotrexate.info
 buymichaelkorsoutletca.ca
@@ -21403,6 +21415,7 @@ c6h12o6.gq
 c6h12o6.ml
 c6h12o6.tk
 c6loaadz.ru
+c70321.com
 c70456.com
 c70567.com
 c70654.com
@@ -24467,6 +24480,7 @@ civilokant903.gq
 civilokant903.ml
 civilokant903.tk
 civilroom.com
+civilyze.com
 civoo.com
 civvic.ro
 civx.org
@@ -29701,6 +29715,7 @@ designfox.org
 designfreebies.ru
 designfuture.info
 designingenium.com
+designingireland.com
 designingknights.com
 designland.info
 designmint.site
@@ -30766,6 +30781,7 @@ discardmail.com
 discardmail.de
 discardmail.ml
 discardmail.tk
+discdots.com
 discfoo.com
 discjumsa.tk
 disckracuvna.cf
@@ -32166,6 +32182,7 @@ drawing-new.ru
 drawinginfo.ru
 drawingwithjeffrey.com
 drawme.site
+drawmeans.com
 drawmedia.org
 drawquest.net
 drazen.dev
@@ -36305,6 +36322,7 @@ evaluatorapp.com
 evalusave.com
 evamail.com
 evanackmann.net
+evandro.cd
 evanfox.info
 evanhamilton.buzz
 evanlifesciences.com
@@ -40678,6 +40696,7 @@ fullen.in
 fullepisodesnow.com
 fullermail.men
 fullfilmizle2.com
+fullframedesign.com
 fullhomepacks.info
 fulll.stream
 fullmoviesonline.space
@@ -41415,6 +41434,7 @@ gangkindter.ga
 gangkindter.gq
 gangkindter.tk
 gangstersofharlem.com
+gangtrk.site
 gangu.cf
 gangu.gq
 gangu.ml
@@ -41730,6 +41750,7 @@ gdian43.com
 gdian49.com
 gdian54.com
 gdian57.com
+gdian63.com
 gdian78.com
 gdian84.com
 gdian86.com
@@ -41743,6 +41764,7 @@ gdmail.top
 gdmalls.com
 gdprcompliance.expert
 gdprcompliance.solutions
+gdqdaintb.ml
 gdqdp2.us
 gdqzhv.com
 gdradr.com
@@ -41996,6 +42018,7 @@ geokomponent.ru
 geolocalroadmap.com
 geologgdpo.space
 geologicpublications.net
+geologik.biz
 geomail.win
 geometricescape.com
 georaster.info
@@ -42195,6 +42218,7 @@ getit-beauty.com
 getitfast.com
 getjar.pl
 getjulia.com
+getkitchen.club
 getkl.site
 getknot.app
 getladiescoats.com
@@ -42370,6 +42394,7 @@ gg023.space
 gg024.space
 gg025.space
 gg18269.com
+gg2srxzj.com
 gg57822.com
 gg7665.com
 gg9094.com
@@ -42585,6 +42610,7 @@ gigism.com
 gign-40.com
 gigpurchase.com
 gihybe.us
+giize.com
 gikemart.site
 gikmail.com
 gilaayam.com
@@ -46515,6 +46541,7 @@ heangdilly.icu
 heari.net
 hearingaiddoctor.net
 hearpower.us
+heart1.ga
 heartandhopefund.com
 heartburnnomorereview.info
 heartfair.xyz
@@ -49679,6 +49706,7 @@ idahowine.tours
 idat.buzz
 idat.site
 idbloc.co
+idcbill.com
 idcclub88.com
 iddaairan.website
 iddaalabayi.com
@@ -50286,6 +50314,7 @@ imagictech.com
 imaginegiants.com
 imagineinvestments.com
 imaginethelearning.com
+imagiscape.photography
 imagiscape.photos
 imagiscape.pictures
 imagiscape.us
@@ -50896,6 +50925,7 @@ infoportalchik.ru
 infoprediksi4d.com
 infoprice.tech
 infoqq24jam.xyz
+inform-factory-new.info
 inform-mart-deal.ru
 inform-new-like.ru
 inform-store-active.ru
@@ -52664,6 +52694,7 @@ jaaj.cf
 jaalaa4.xyz
 jaanv.com
 jaarvandemolens.online
+jaat.live
 jabatankopi.com
 jabberflash.info
 jabbyhotel.online
@@ -56576,6 +56607,7 @@ kmeuktpmh.pl
 kmhow.com
 kmkl.de
 kmlueh.shop
+kmonger.co
 kmonkeyd.com
 kmonlinestore.co.uk
 kmqqu.live
@@ -59583,6 +59615,7 @@ lightspeed.store
 lightspeedgolf.app
 lightswea.icu
 lightvid.ru
+lightvids.com
 lightvivo.ru
 lighvanrau.ga
 ligirls.ru
@@ -66894,6 +66927,7 @@ moldered494dn.online
 moldova-nedv.ru
 moldova.uno
 moldujgkcqb.email
+moldura888.com
 molecadamail.pw
 molkq.site
 moll.express
@@ -70270,6 +70304,7 @@ new-year-lucky-gifts.monster
 new-year-special-gift.icu
 new-year-special-gift.monster
 new-york-wedding.com
+new-york.host
 new688e.ga
 newa.wtf
 newage.press
@@ -79282,6 +79317,7 @@ preccompdes.gq
 preccompdes.ml
 precconta.cf
 precconta.ga
+precconta.tk
 precdate.cf
 precdate.ga
 precdate.gq
@@ -87769,6 +87805,7 @@ sciegenics.com
 science-aviation.org
 science-full.ru
 scienceacademicnews.site
+scienceauvert.org
 sciencelive.ru
 sciencepub.news
 sciencestill.com
@@ -95751,6 +95788,7 @@ teamandclub.ga
 teambogor.online
 teamdigest.com
 teamflow.works
+teamhv.net
 teamjulie.com
 teamkg.tk
 teamkiller.net
@@ -95769,6 +95807,7 @@ teamspeak3.ga
 teamster.com
 teamtimspencer.com
 teamtournamenttravel.com
+teamtriton.net
 teamviewerindirsene.com
 teamwyden.net
 teamxpress.org
@@ -95930,6 +95969,7 @@ tee800.com
 teebaum-oel.info
 teecheap.store
 teedinnan.com
+teeessential.com
 teemia.com
 teemo.site
 teemoco.shop
@@ -97859,6 +97899,7 @@ thyroidsaver.org
 thyroidtips.info
 thyxwenu.shop
 thzhhe5l.ml
+ti-sale.online
 tiabami.cf
 tiabami.ga
 tiabami.gq
@@ -104424,6 +104465,7 @@ viddly.online
 viddly.plus
 viddly.site
 viddsy.net
+videgertools.info
 video-16porno.fr
 video-dlj-tebya.ru
 video-insanity.com
@@ -104849,6 +104891,7 @@ viraginous.xyz
 viral-hub.xyz
 viral-science.fun
 viral-update.xyz
+viralchoose.com
 viralclothes.com
 viralhits.org
 virallifestyles.com
@@ -106438,6 +106481,7 @@ warnerbrobrewco.com
 warnerwave.xyz
 warning-10by25.stream
 warnomics.com
+waroengdo.store
 waronrent.org
 warp2p.com
 warqaat.com
@@ -106742,6 +106786,7 @@ wdrecipes.com
 wdrzw.info
 wdsfbghfg77hj.gq
 wdvhw1.site
+wdw.ru
 wdwot.com
 wdx4.com
 wdxgc.com
@@ -107417,6 +107462,7 @@ wgfvim.us
 wgkfdq.host
 wgkoeln.org
 wgnyw.info
+wgraj.com
 wgrazd.site
 wgw365.com
 wgweddxy.shop
@@ -110756,6 +110802,7 @@ xn--rgfest-vxa4c.com
 xn--rhqt5tmrbt30afkiqpk45b.top
 xn--ridsp-nua.se
 xn--robotjurdico-zfb.com
+xn--rpple-tsa.com
 xn--rrmokarejnkping-8sbic.nu
 xn--rrmokarenorrkping-zzbm.com
 xn--saatbahaber-4zb11d.com