email_data 1605293459 → 1605671224

data/VERSION

@@ -1 +1 @@
-1605293459
+1605671224

data/bin/console

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 require "bundler/setup"
 require "email_data"
@@ -6,9 +7,10 @@ require "email_data"
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.
 
-# (If you use this, don't forget to add pry to your Gemfile!)
-# require "pry"
-# Pry.start
-
-require "irb"
-IRB.start(__FILE__)
+begin
+  require "pry"
+  Pry.start
+rescue LoadError
+  require "irb"
+  IRB.start(__FILE__)
+end

data/bin/publish-npm

@@ -7,12 +7,6 @@ source $HOME/.zsh/asdf.sh
 
 asdf local nodejs 12.18.4
 
-for input in $(ls -1 data/*.txt); do
-  name=$(basename $input)
-  output=data/json/"${name%.txt}.json"
-  jq -R -s -c 'split("\n")' < $input > $output
-done
-
 npm publish --access=public
 
 terminal-notifier \

data/bin/setup

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
+
 set -euo pipefail
+
 IFS=$'\n\t'
 set -vx
 

data/bin/sync

@@ -5,3 +5,5 @@ echo
 ./bin/sync-free-emails
 echo
 ./bin/sync-disposable-emails
+echo
+./bin/export-json-files

data/bin/sync-disposable-emails

@@ -182,6 +182,10 @@ threads.clear
 
 domains = []
 
+puts "=> Saving private relays"
+private_relays = normalize_list(File.read("#{__dir__}/../data/manual/private_relays.txt").lines)
+save_file("private_relays.txt", private_relays)
+
 puts "=> Loading disposable_domains.txt"
 domains += normalize_list(File.read("#{__dir__}/../data/disposable_domains.txt").lines)
 
@@ -196,9 +200,8 @@ end
 
 ignore_domains = normalize_list(File.read("#{__dir__}/../data/free_email_domains.txt").lines)
                  .map {|domain| RootDomain.call(domain) }
-ignore_domains += %w[
-  appleid.com
-]
+ignore_domains += normalize_list(File.read("#{__dir__}/../data/private_relays.txt").lines)
+                  .map {|line| RootDomain.call(line) }
 
 puts "=> Normalize domains (count: #{domains.size})"
 domains = domains

data/bin/sync-tld

@@ -26,3 +26,18 @@ country_tlds = country_tlds
 
 puts "=> Saving country_tlds.txt"
 save_file("country_tlds.txt", normalize_list(country_tlds))
+
+puts "=> Fetching sld list"
+response = http_request(:get, "https://publicsuffix.org/list/public_suffix_list.dat")
+content, _ = response.body.split("END ICANN DOMAINS")
+
+slds = content
+       .lines
+       .map {|line| line.gsub(/^\*\./, "") }
+       .map(&:strip)
+       .reject(&:empty?)
+       .reject {|line| line.start_with?("/") }
+       .map {|line| line.gsub(/^([!*.]+)/, "") }
+       .reject {|line| line.split(".").size == 1 }
+
+save_file("slds.txt", normalize_list(slds))

data/data/disposable_domains.txt

@@ -33,6 +33,7 @@
 001xs.net
 001xs.org
 001xs.xyz
+002.city
 002288211.com
 002r.com
 002t.com
@@ -6988,6 +6989,7 @@
 8verxcdkrfal61pfag.tk
 8vkopg.us
 8vnond.us
+8w903.buzz
 8wehgc2atizw.cf
 8wehgc2atizw.ga
 8wehgc2atizw.gq
@@ -7027,6 +7029,7 @@
 8xgui.com
 8xgun.com
 8xgve.com
+8xi.org
 8xzr.com
 8ycxiazai.com
 8ycxz.com
@@ -7773,6 +7776,7 @@ a02sjv3e4e8jk4liat.gq
 a02sjv3e4e8jk4liat.ml
 a02sjv3e4e8jk4liat.tk
 a0f7ukc.com
+a0fnjd.us
 a0reklama.pl
 a0txa.icu
 a0ywhm.us
@@ -8374,6 +8378,7 @@ accumolecular.com
 accur8.net
 accuracyis.com
 accuranker.tech
+accuratecallabs.com
 accuratecallabs.net
 accurateforum.info
 accuratehealthman.xyz
@@ -10214,6 +10219,7 @@ alcsehar.ml
 alcsehar.tk
 alcsx.site
 alcyonoid.info
+aldawaeya.org
 aldealdi.shop
 aldeaminera.info
 aldemimea.xyz
@@ -10358,6 +10364,7 @@ alhadattv.com
 alhalfpricelistings.com
 alhamadealmeria.com
 alhashareslectronics.com
+alhemyaria.news
 alhzfw.us
 ali-baba.info
 aliagaberrakemlak.com
@@ -10431,6 +10438,7 @@ alisverisistan.net
 alitaj.com
 alittle.website
 alivance.com
+aliveinlosangeles.com
 alivemail.cf
 alivemail.ga
 alivemail.gq
@@ -14838,7 +14846,9 @@ bahisstar221.com
 bahisstar226.com
 bahisstar228.com
 bahisstar229.com
+bahistek39.com
 bahistek40.com
+bahistek86.com
 bahisturk14.com
 bahisturk17.com
 bahisturk23.com
@@ -16999,6 +17009,7 @@ betnis87.com
 betnis89.com
 betodesa.com
 betoffice206.com
+betoffice224.com
 betofis.net
 betofis10.com
 betofis101.com
@@ -17941,6 +17952,7 @@ birminghamalcoholrehab.com
 birminghamcocainerehab.com
 birminghamfans.com
 birminghamheroinrehab.com
+birminghamquote.com
 biro.gq
 biro.ml
 biro.tk
@@ -18441,6 +18453,7 @@ blogcash.com
 blogcast.blog
 blogcast.lgbt
 blogcast.us
+blogchannels.com
 blogcollege.com
 blogdad.com
 blogdiary.live
@@ -19899,6 +19912,7 @@ brisbanephotobooth.com
 bristlebrush.us
 bristlebrushes.biz
 bristlebrushes.us
+bristol-drones.com
 britainst.com
 britbarnmi.ga
 britbarnmi.gq
@@ -20079,6 +20093,7 @@ brvbfj.shop
 brxqoy.cf
 bryanle.com
 bryanlgx.com
+bryanslist.com
 brymstonne.org
 brypeterson.com
 bryskumde.cf
@@ -20676,6 +20691,7 @@ buvertu.cf
 buvertu.ga
 buvertu.gq
 buvertu.ml
+buvetti.com
 buxap.com
 buxl.com
 buxod.com
@@ -20779,6 +20795,7 @@ buylaptopsunder300.com
 buylevitra-us.com
 buylevitra.website
 buylikes247.com
+buylimu.biz
 buylouisvuittonbagsjp.com
 buymethotrexate.info
 buymichaelkorsoutletca.ca
@@ -21061,6 +21078,7 @@ buyyourtoledohome.com
 buyyoutubviews.com
 buzlat.org
 buzlin.club
+buzz.buzz
 buzzcluby.com
 buzzcompact.com
 buzzdating.info
@@ -22361,6 +22379,7 @@ casinobit.club
 casinobonusual2.com
 casinochan2.com
 casinocuan.club
+casinoelit47.com
 casinoeridanmark.com
 casinofundaily.info
 casinogreat.club
@@ -23621,6 +23640,7 @@ cheapukniketrainerssale.co.uk
 cheapuksalehandbagsoutletlv.co.uk
 cheapukstraightenerssale.info
 cheapusbspeakers.info
+cheapviagra.org
 cheapwebtraffic.com
 cheapweekendgetawaysforcouples.com
 cheatautomation.com
@@ -24451,6 +24471,7 @@ citymdisp.ga
 citymdisp.gq
 citymdisp.ml
 cityofnah.com
+cityquote.com
 cityroyal.org
 citytorate.ru
 citytowercasino.com
@@ -24851,6 +24872,7 @@ clickmp.pro
 clickmpnew.pro
 clicknett.com
 clicknowk.pro
+clickp.pro
 clickpet.online
 clickpm.pro
 clickpnow.pro
@@ -25627,12 +25649,14 @@ coloshild.ru
 coloshron.ru
 colosjoy.ru
 colossuscloud.us
+colourandcode.com
 colt45forsale.tk
 coltonattorneys.com
 columbuscasino.design
 columbuseldercare.com
 columbusinfo.xyz
 columbusnm.net
+columbusquote.com
 com-04536875422364784521.top
 com-101634586696.help
 com-1024493459167.online
@@ -25669,6 +25693,8 @@ com-xvrv6yt51g.com
 com.com
 com.mz
 com.ninja
+com.np
+com.ru
 com.uk
 com.us
 com3b.com
@@ -25878,6 +25904,7 @@ computerslistnews.site
 computersoftware2012.info
 computerspeakers22.com
 computersshop.futbol
+computingzone.org
 compvershy.cf
 compvershy.ga
 compvershy.gq
@@ -27166,6 +27193,7 @@ crudeoildemulsifier.com
 crudewag.icu
 cruisecentraladviser.com
 cruisefloppy.com
+cruisessale.com
 crulle.ru
 crumlin-grill.com
 crunchcompass.com
@@ -28310,6 +28338,7 @@ dalamananadolulisesi.xyz
 dalamanporttransfer.xyz
 dalamanturkishvillas.xyz
 dalanila.tk
+dalatrk.site
 dalatvirginia.com
 dalbam7.net
 daleadershipinstitute.org
@@ -28828,6 +28857,7 @@ db4t5tes4.ga
 db4t5tes4.gq
 db4t5tes4.ml
 db4t5tes4.tk
+dbadhe.com
 dbanote.net
 dbatalk.com
 dbataturkioo.com
@@ -29712,6 +29742,7 @@ designfox.org
 designfreebies.ru
 designfuture.info
 designingenium.com
+designingireland.com
 designingknights.com
 designland.info
 designmint.site
@@ -36334,6 +36365,7 @@ evavbe.ml
 evavbe.tk
 evavoyance.com
 evbank.ru
+evbholdingsllc.com
 evcmail.com
 evcr8twoxifpaw.cf
 evcr8twoxifpaw.ga
@@ -36799,6 +36831,7 @@ extra-penis-enlargement.info
 extraaaa.tk
 extraaaa2.ga
 extraaaa2.tk
+extraale.com
 extraam.loan
 extrabooks.site
 extracccolorrfull.com
@@ -36831,6 +36864,7 @@ exttract.email
 extureme.com
 exwta.us
 exxale.shop
+exxale.space
 exxon-mobil.tk
 exxoncars.com
 exxx.club
@@ -42057,6 +42091,7 @@ gerandos.ru
 gerdese.online
 gerdimenta.ru
 geremail.info
+gerenc.host
 gerenciagran.com
 gergilimembran.com
 gerhanajitu.net
@@ -42315,6 +42350,7 @@ getyougadgets.com
 getyourbook.site
 getyourstudy.ru
 getyourtantraon.com
+getzradonpros.net
 geuur.live
 gevemarket.site
 gewinner-pose.com
@@ -44497,6 +44533,7 @@ groupe-psa.cf
 groupe-psa.gq
 groupe-psa.ml
 groupe-psa.tk
+groupe.cd
 groupfunds.club
 groupkami.com
 groupmaidat.com
@@ -49850,6 +49887,7 @@ ieattach.ml
 iecrater.com
 iecusa.net
 iedindon.ml
+ieeeves.org
 iefbcieuf.cf
 iefbcieuf.ml
 iefbcieuf.tk
@@ -50310,6 +50348,7 @@ imagictech.com
 imaginegiants.com
 imagineinvestments.com
 imaginethelearning.com
+imagiscape.photography
 imagiscape.photos
 imagiscape.pictures
 imagiscape.us
@@ -50824,7 +50863,6 @@ infinityangelhealingpathways.com
 infinitybooksjapan.org
 infinityclasses.info
 infinityclippingpath.com
-infis.net.ru
 infitter.ru
 inflammationpills.info
 inflatableslife.com
@@ -57378,6 +57416,7 @@ kuatocokjaran.gq
 kuatocokjaran.ml
 kuatocokjaran.tk
 kuba-nedv.ru
+kubeflow.info
 kubqs.live
 kubzone.ru
 kuchenmobel-berlin.ovh
@@ -57855,6 +57894,7 @@ labogili.ga
 labontemty.com
 laboralistascoruna.com
 laboratortehnicadentara.ro
+laboratoryreport.win
 labored673vl.online
 laboriously.com
 laborstart.org
@@ -59027,6 +59067,7 @@ lesabahis40.com
 lesabahis48.com
 lesabahis49.com
 lesabahis55.com
+lesabahis90.com
 lesastroi.ru
 lesbugs.com
 lesfineslamesdechicoutimi.com
@@ -61816,7 +61857,6 @@ lyaliume.site
 lyamda.ru
 lybaba.xyz
 lybe.info
-lyceum-life.com.ru
 lydia-uniform.com
 lydias-scrubs.com
 lydiascrubs.com
@@ -62042,6 +62082,7 @@ mabterssur.ml
 mabterssur.tk
 mabuklagi.ga
 mabulareserve.com
+mabv.club
 macaniuo235.cf
 macankumbang.com
 macaoguojihui.com
@@ -65472,6 +65513,7 @@ mft9864784.xyz
 mft9867978.xyz
 mft9876343.xyz
 mft9883173.xyz
+mft9909394.xyz
 mft9911897.xyz
 mft9920868.xyz
 mfuil.us
@@ -66795,6 +66837,7 @@ modetoxcenter.com
 modety.online
 modhack.net
 modicadacademy.com
+modikulp.com
 modish.net
 modjunkies.com
 modkjaj.site
@@ -67282,6 +67325,7 @@ mormoncoffee.com
 mornayoovm.space
 mornhfas.org.ua
 morningstarkafe.xyz
+morningstiffnesspodcast.org
 morningtw.com
 mornsoft.com
 morogamers.com
@@ -67645,6 +67689,7 @@ mrchinh.com
 mrclipper.com
 mrcraftyconsultant.com
 mrctacoma.com
+mrdeeps.ml
 mrdevilstore.com
 mrdjg.live
 mrdomino99.org
@@ -69332,7 +69377,6 @@ namste99.com
 namtinh.top
 namtovarovedam.ru
 namtruong318.com
-namunathapa.com.np
 namuoutlets.site
 namushops.site
 namuwikiusercontent.com
@@ -70069,6 +70113,7 @@ net-piyango.biz
 net-privichkam.ru
 net-solution.info
 net.com
+net.ru
 net191.com
 net1mail.com
 net2222.com
@@ -70299,6 +70344,7 @@ new-year-lucky-gifts.monster
 new-year-special-gift.icu
 new-year-special-gift.monster
 new-york-wedding.com
+new-york.host
 new688e.ga
 newa.wtf
 newage.press
@@ -74649,6 +74695,7 @@ osfujhtwrblkigbsqeo.ml
 osfujhtwrblkigbsqeo.tk
 osg168live.asia
 osg168live.us
+osg777.link
 osgame.live
 osgame.online
 osgpoker.us
@@ -75335,6 +75382,7 @@ p6udt9.us
 p71ce1m.com
 p7665.com
 p7998.com
+p7n39rellip7.best
 p8oan2gwrpbpvbh.cf
 p8oan2gwrpbpvbh.ga
 p8oan2gwrpbpvbh.gq
@@ -77407,6 +77455,7 @@ pidcockmarketing.com
 pidemudanza.com
 pidmail.com
 pidoseme.online
+pidouno.com
 pidox.org
 piebombing.com
 piececoun.icu
@@ -77586,6 +77635,7 @@ pinsonian.org
 pinsup.ru
 pintaresfacilconsapolin.com
 pintasticapp.com
+pintech.ru
 pintechpinball.com
 pinterest-seo.online
 pinterests.pro
@@ -79054,7 +79104,6 @@ powerhousedyno.com
 powerin.site
 powerlea.xyz
 powerlear.xyz
-powerlink.com.np
 powermass.site
 powerml.racing
 powerpages.website
@@ -79582,6 +79631,7 @@ prilution.info
 prim1v.us
 primabananen.net
 primalburnkenburge.com
+primaryale.com
 primaryvideos.com
 primasecure.cd
 primate.de
@@ -81612,6 +81662,7 @@ quanaril.cf
 quanaril.com
 quanaril.ga
 quanaril.ml
+quandahui.com
 quangcaoso1.net
 quant-heyr.pro
 quant-rr.pro
@@ -82302,6 +82353,7 @@ ragzwtna4ozrbf.tk
 rahabionic.com
 rahavpn.men
 rahimis.com
+rahul.cyou
 raiasu.cf
 raiasu.ga
 raiasu.gq
@@ -86170,6 +86222,7 @@ rvb.ro
 rvbspending.com
 rvctf.com
 rvdogs.com
+rvemold.com
 rviixj.us
 rvjtudarhs.cf
 rvjtudarhs.ga
@@ -87539,6 +87592,7 @@ savethechildrenactionnetwork.com
 savetimeerr.fun
 saveuhealth.com
 savevid.ga
+saveyourgadget.com
 savingallhomes.com
 savingnyhomes.com
 savingsearcher.com
@@ -87586,6 +87640,7 @@ saymeow.de
 saymuscge.cf
 saymuscge.ga
 saymuscge.ml
+saynigger.com
 saynotospams.com
 sayonara.gq
 sayonara.ml
@@ -87905,6 +87960,7 @@ screenvel.com
 screwdon.recipes
 screwdriveraction.com
 screwthe.xyz
+scribb.eu
 scribble.uno
 scribeorigins.org
 script.click
@@ -90097,6 +90153,7 @@ simplesport.ru
 simpletextmarketing.com
 simpleverification.com
 simpleverni.icu
+simplexion.pm
 simplictylegal.com
 simplus.com.br
 simply-email.bid
@@ -95963,6 +96020,7 @@ tee800.com
 teebaum-oel.info
 teecheap.store
 teedinnan.com
+teeessential.com
 teemia.com
 teemo.site
 teemoco.shop
@@ -96512,6 +96570,7 @@ test130.com
 test324t6234w5y.ru
 testa-lot.net
 testadobe.ru
+testbnk.com
 testclean.org
 testclubs.com
 testdom34533663.host
@@ -98161,6 +98220,7 @@ tintuceva.org
 tintucphunu.org
 tinviahe.top
 tinybt.com
+tinyheight.com
 tinymill.org
 tinypc.reviews
 tinypc.tech
@@ -101656,6 +101716,7 @@ ucq9vbhc9mhvp3bmge6.gq
 ucq9vbhc9mhvp3bmge6.ml
 ucr6pb.site
 ucroyal09.com
+ucsoft.biz
 uctqoj.us
 uctwh.us
 ucupdong.ml
@@ -104218,6 +104279,7 @@ vestgreensig.cf
 vestgreensig.ga
 vestgreensig.gq
 vestgreensig.tk
+vestigeschool.com
 vestimed.ru
 vestnikao.ru
 vestniktm.ru
@@ -104350,6 +104412,7 @@ vhntp15yadrtz0.gq
 vhntp15yadrtz0.ml
 vhntp15yadrtz0.tk
 vhobbi.ru
+vhoff.com
 vhpsc.us
 vhw5ie.us
 vhwiz8.us
@@ -105444,6 +105507,7 @@ volumetudo.website
 volumkas.cf
 volunteergarden.org
 voluptuary.site
+volvefloor.com
 volvo-ab.cf
 volvo-ab.ga
 volvo-ab.gq
@@ -110795,6 +110859,7 @@ xn--rgfest-vxa4c.com
 xn--rhqt5tmrbt30afkiqpk45b.top
 xn--ridsp-nua.se
 xn--robotjurdico-zfb.com
+xn--rpple-tsa.com
 xn--rrmokarejnkping-8sbic.nu
 xn--rrmokarenorrkping-zzbm.com
 xn--saatbahaber-4zb11d.com
@@ -111143,6 +111208,7 @@ xtremeventure.com
 xtremewebtraffic.net
 xtrstories.com
 xtrstudios.com
+xtrtarget.com
 xtrzeer.fun
 xtubemate.site
 xtuc.com
@@ -111695,7 +111761,6 @@ yachassa09.com
 yachassa16.com
 yackir.store
 yacxrz.pl
-yadavnaresh.com.np
 yadegarigift.icu
 yadkincounty.org
 yadong4.com