email_data 1606845461 → 1607427861

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cc3775652922ba0bb5fb85561a06a71a2447e1ddfb0c6787dcae43e21296a4e8
-  data.tar.gz: e3311f3350eb7f5399b9a414d6f45f3302c2d1b9fc41e6b8b2d6695596447105
+  metadata.gz: 63b2324c3c5b0b6cf82035d848fdc7bb4da8ddd7f4e5fd2ea5bacff09eb37fe0
+  data.tar.gz: 42505ee545597463a44604089b0d8288daee2a5b4bbddc237dbe9d6e20c59c0f
 SHA512:
-  metadata.gz: 6797f349be0ffd4bc7c78e9b15d23e03fc72b67b6e49bc2b54f022c96d88cabaed1dd32c1c4d1dbd2ac27ac5c89e6de744f91c72021d6390983562a3ea380483
-  data.tar.gz: 36e08df08cb90cc60c7afc9fd1e6d17fc26aae3254a63abb816685371e92c395d8306d000a04ba832b05a0267a8612a4ec81cd6697fb285ea8c8a4d25443098e
+  metadata.gz: f46c3cc8c6760ca863e8d70804f48b6badf714368815cce42a8ce12c386849e79e1f8996cf4c1168cba143ca8360797a4463b75f8921416e6b7405f412cffe3b
+  data.tar.gz: dc79df9eecce6e95e44c15d806668ce585ee8147bc8ac94ea614e40e1992a3310d66a877ee95d3c8f67b933623532ea0b1ecc4651438386cf6da7e1d0ebc3881

data/.github/FUNDING.yml

@@ -1,4 +1,3 @@
-# These are supported funding model platforms
-
+---
 github: [fnando]
 custom: ["https://paypal.me/nandovieira/🍕"]

data/.github/workflows/package.yml

@@ -0,0 +1,49 @@
+---
+name: package
+
+on:
+  schedule:
+    - cron: "0 0 * * *"
+  workflow_dispatch:
+    inputs: {}
+
+jobs:
+  build:
+    name: Export with Ruby ${{ matrix.ruby }}
+    runs-on: "ubuntu-latest"
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: ["2.7.x"]
+
+    steps:
+      - uses: actions/checkout@v2.3.4
+
+      - uses: actions/cache@v2
+        with:
+          path: vendor/bundle
+          key: >
+            ${{ runner.os }}-${{ matrix.ruby }}-gems-${{
+            hashFiles('**/*.gemspec') }}
+          restore-keys: >
+            ${{ runner.os }}-${{ matrix.ruby }}-gems-${{
+            hashFiles('**/*.gemspec') }}
+
+      - name: Set up Ruby
+        uses: actions/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+
+      - name: Install gem dependencies
+        run: |
+          gem install bundler
+          bundle config path vendor/bundle
+          bundle install --jobs 4 --retry 3
+
+      - name: Export files
+        env:
+          GITHUB_TOKEN: "${{ secrets.GH_TOKEN }}"
+          RUBYGEMS_API_KEY: "${{ secrets.RUBYGEMS_API_KEY }}"
+          NPM_AUTH_TOKEN: "${{ secrets.NPM_AUTH_TOKEN }}"
+        run: |
+          ./bin/package

data/.github/workflows/ruby-tests.yml

@@ -1,3 +1,4 @@
+---
 name: ruby-tests
 
 on:

data/.gitignore

@@ -9,3 +9,24 @@
 /data/disposable/**/*
 /data/json/**/*
 LOCK
+bin/jq
+opt/
+vendor/
+pkg/
+VERSION
+.DS_Store
+.*.sw?
+/.pow*
+/.rvmrc
+/.rbenv-version
+/bin/
+/.tags*
+Thumbs.db
+.vagrant
+.bash/user.sh
+.gemtags
+.tags
+.tags_sorted_by_file
+/node_modules
+/.env
+/.env.*

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    email_data (1606845461)
+    email_data (1607309899)
 
 GEM
   remote: https://rubygems.org/
@@ -20,10 +20,14 @@ GEM
     aitch (1.1.0)
       nokogiri
     ast (2.4.1)
+    awesome_print (1.8.0)
+    byebug (11.1.3)
+    coderay (1.1.3)
     concurrent-ruby (1.1.7)
     docile (1.3.2)
     i18n (1.8.5)
       concurrent-ruby (~> 1.0)
+    method_source (1.0.0)
     mini_portile2 (2.4.0)
     minitest (5.14.2)
     minitest-utils (0.4.6)
@@ -34,6 +38,20 @@ GEM
     parser (2.7.2.0)
       ast (~> 2.4.1)
     pg (1.2.3)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    pry-byebug (3.9.0)
+      byebug (~> 11.0)
+      pry (~> 0.13.0)
+    pry-meta (0.0.10)
+      awesome_print
+      pry
+      pry-byebug
+      pry-remote
+    pry-remote (0.1.8)
+      pry (~> 0.9)
+      slop (~> 3.0)
     rainbow (3.0.0)
     rake (13.0.1)
     regexp_parser (2.0.0)
@@ -41,18 +59,18 @@ GEM
     root_domain (0.1.2)
       email_data
       simpleidn
-    rubocop (1.5.0)
+    rubocop (1.5.2)
       parallel (~> 1.10)
       parser (>= 2.7.1.5)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 2.0)
+      regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 1.2.0)
+      rubocop-ast (>= 1.2.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 2.0)
     rubocop-ast (1.3.0)
       parser (>= 2.7.1.5)
-    rubocop-fnando (0.0.14)
+    rubocop-fnando (0.0.15)
       rubocop
       rubocop-performance
     rubocop-performance (1.9.1)
@@ -67,6 +85,7 @@ GEM
     simplecov_json_formatter (0.1.2)
     simpleidn (0.1.1)
       unf (~> 0.1.4)
+    slop (3.6.0)
     thread_safe (0.3.6)
     tzinfo (1.2.8)
       thread_safe (~> 0.1)
@@ -87,6 +106,7 @@ DEPENDENCIES
   minitest
   minitest-utils
   pg
+  pry-meta
   rake
   root_domain
   rubocop

data/README.md

@@ -41,6 +41,8 @@ EmailData.data_dir
 
 # List of disposable domains. Punycode is expanded into ASCII domains.
 EmailData.disposable_domains
+EmailData.disposable_domains_with_mx
+EmailData.disposable_domains_without_mx
 
 # List of disposable emails. Some services use free email like Gmail to create
 # disposable emails.
@@ -127,6 +129,18 @@ class SetupEmailData < ActiveRecord::Migration[6.1]
 
     add_index :disposable_domains, :name, unique: true
 
+    create_table :disposable_domains_with_mx do |t|
+      t.citext :name, null: false
+    end
+
+    add_index :disposable_domains_with_mx, :name, unique: true
+
+    create_table :disposable_domains_without_mx do |t|
+      t.citext :name, null: false
+    end
+
+    add_index :disposable_domains_without_mx, :name, unique: true
+
     create_table :free_email_domains do |t|
       t.citext :name, null: false
     end
@@ -176,6 +190,8 @@ COPY slds (name) FROM '/usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0/gems/email_data
 COPY country_tlds (name) FROM '/usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0/gems/email_data-1601479967/data/country_tlds.txt';
 COPY disposable_emails (name) FROM '/usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0/gems/email_data-1601479967/data/disposable_emails.txt';
 COPY disposable_domains (name) FROM '/usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0/gems/email_data-1601479967/data/disposable_domains.txt';
+COPY disposable_domains_with_mx (name) FROM '/usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0/gems/email_data-1601479967/data/disposable_domains_with_mx.txt';
+COPY disposable_domains_without_mx (name) FROM '/usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0/gems/email_data-1601479967/data/disposable_domains_without_mx.txt';
 COPY free_email_domains (name) FROM '/usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0/gems/email_data-1601479967/data/free_email_domains.txt';
 COPY roles (name) FROM '/usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0/gems/email_data-1601479967/data/roles.txt';
 COPY private_relays (name) FROM '/usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0/gems/email_data-1601479967/data/private_relays.txt';
@@ -204,6 +220,8 @@ class LoadEmailData < ActiveRecord::Migration[6.1]
     copy.call(:country_tlds)
     copy.call(:disposable_emails)
     copy.call(:disposable_domains)
+    copy.call(:disposable_domains_with_mx)
+    copy.call(:disposable_domains_without_mx)
     copy.call(:free_email_domains)
     copy.call(:roles)
     copy.call(:private_relays)
@@ -230,6 +248,8 @@ $ npm install @fnando/email_data
 ```js
 const disposableEmails = require("@fnando/email_data/data/json/disposable_emails.json");
 const disposableDomains = require("@fnando/email_data/data/json/disposable_domains.json");
+const disposableDomainsWithMx = require("@fnando/email_data/data/json/disposable_domains_with_mx.json");
+const disposableDomainsWithoutMx = require("@fnando/email_data/data/json/disposable_domains_without_mx.json");
 const freeEmailDomains = require("@fnando/email_data/data/json/free_email_domains.json");
 const roles = require("@fnando/email_data/data/json/roles.json");
 const privateRelays = require("@fnando/email_data/data/json/private_relays.json");

data/Rakefile

@@ -2,7 +2,8 @@
 
 require "bundler/gem_tasks"
 require "rake/testtask"
-require "rubocop/rake_task"
+
+all_tasks = %i[test]
 
 Rake::TestTask.new(:test) do |t|
   t.libs << "test"
@@ -10,6 +11,12 @@ Rake::TestTask.new(:test) do |t|
   t.test_files = FileList["test/**/*_test.rb"]
 end
 
-RuboCop::RakeTask.new
+begin
+  require "rubocop/rake_task"
+  RuboCop::RakeTask.new
+  all_tasks << "rubocop"
+rescue LoadError
+  puts "=> Couldn't load Rubocop rake tasks"
+end
 
-task default: %i[test rubocop]
+task default: all_tasks

data/VERSION

@@ -1 +1 @@
-1606845461
+1607200035

data/bin/helpers.rb

@@ -106,9 +106,9 @@ def random_timeout
   (1..10).map {|i| (1.0 / i).round(2) }.sample
 end
 
-def refresh_list(url:, path:, verb: :get, params: nil, &block)
+def refresh_list(url:, path:, verb: :get, params: nil)
   response = http_request(verb, url, params)
-  items = block.call(response)
+  items = yield(response)
   append_to_file(path, items)
 rescue StandardError => error
   puts "=> [ERROR] Unable to refresh list: #{url} (#{error.class}: #{error.message})"
@@ -191,9 +191,9 @@ def root_domains(domains)
   processed_domains.reject(&:nil?)
 end
 
-def thread(&block)
+def thread
   Thread.new do
-    block.call
+    yield
   rescue StandardError => error
     puts "=> [ERROR] Thread has errored; #{error.class}: #{error.message}"
   end

data/bin/publish

@@ -8,17 +8,12 @@ source $HOME/.zsh/asdf.sh
 lockfile=$(dirname $0)/../LOCK
 
 if [[ -f "$lockfile" ]]; then
-  terminal-notifier \
-    -message "Skipping script; repository is locked." \
-    -ignoreDnD \
-    -title "email_data" \
-    -group homebrew \
-    &> /dev/null
+  echo "Skipping script; repository is locked."
   exit 1
 fi
 
 git reset --hard
-git checkout main
+git checkout publish
 git pull --rebase
 git push
 
@@ -27,13 +22,8 @@ git push
 git_status=$(git status 2> /dev/null | tr "\\n" " ")
 
 if [[ "$git_status" =~ "nothing to commit" ]]; then
-  terminal-notifier \
-    -message "No changes detected. Skipping new packages." \
-    -ignoreDnD \
-    -title "email_data" \
-    -group homebrew \
-    &> /dev/null
-  exit 1
+  echo "No changes detected. Skipping new packages."
+  exit
 fi
 
 date "+%s" > VERSION
@@ -52,5 +42,5 @@ git add .
 git commit -m "Bump up version."
 git push
 
-./bin/publish-gem
-./bin/publish-npm
+# ./bin/publish-gem
+# ./bin/publish-npm

data/bin/sync

@@ -2,8 +2,12 @@
 
 ./bin/sync-tld
 echo
+./bin/sync-roles
+echo
 ./bin/sync-free-emails
 echo
 ./bin/sync-disposable-emails
 echo
+./bin/filter-disposable-domains-without-mx
+echo
 ./bin/export-json-files

data/bin/sync-disposable-emails

@@ -130,7 +130,7 @@ def process_scraping(element, value_selector)
   else
     element.attributes.each do |_name, attr|
       attr = attr.value.to_s
-      value = attr if attr =~ host_regex
+      value = attr if attr&.match?(host_regex)
     end
   end
 
@@ -172,7 +172,6 @@ threads << thread { domain_scraping("itemp", "https://itemp.email", "select[name
 threads << thread { domain_scraping("cs", "https://www.cs.email", "select[id=gm-host-select] option::text()") }
 threads << thread { domain_scraping("tempmail", "https://tempmail.io/settings/", "select[id=domain] option::text()") }
 threads << thread { domain_scraping("tempemail", "https://tempemail.co", "select[name=email_domain] option::text()") }
-threads << thread { domain_scraping("tmail", "https://mytemp-email.com/", "a.domain-selector::text()") }
 
 threads.each_slice(5) do |slice|
   slice.each(&:join)

data/data/disposable_domains.txt

@@ -1455,6 +1455,7 @@
 1hdbt.com
 1hdd.site
 1hdek9.us
+1heizi.com
 1hermesbirkin0.com
 1hi.info
 1hmoxs72qd.cf
@@ -4588,6 +4589,7 @@
 567107.com
 56787.com
 5679827.com
+567map.xyz
 568ac.com
 56k.tokyo
 56toyk.us
@@ -13340,6 +13342,7 @@ aspfitting.com
 asphalt-9-legends.ru
 aspiredoge.com
 aspiringweb.net
+aspiringweb.org
 asportsa.ru
 asportscoach.app
 aspotgmail.org
@@ -13843,6 +13846,7 @@ aussiematureclub.com
 aussiepersonal.loan
 aussiepersonal.loans
 aussiesmut.com
+austbikemart.com
 austimail.com
 austinbell.name
 austincar.club
@@ -20440,6 +20444,7 @@ brownetrading.net
 brownetrading.org
 browniesgoreng.com
 brownieslumer.com
+brownindians.com
 brownlandscapingutah.com
 brownpelican.info
 brownricecooker.com
@@ -47529,6 +47534,7 @@ hebagfu.tk
 hebatmart.xyz
 hebedbugremovalok.live
 hebeos.online
+hebgsw.com
 hebohdomino88.com
 hebohpkv.design
 hebohpkv2.com
@@ -48629,6 +48635,7 @@ hmmbswlt5ts.ga
 hmmbswlt5ts.gq
 hmmbswlt5ts.ml
 hmmbswlt5ts.tk
+hmnmw.com
 hmos.systems
 hmotomotiv.xyz
 hmsale.org
@@ -54711,6 +54718,7 @@ jiwatoto.org
 jixa.site
 jixe.site
 jixyvu.info
+jiyankotluk.xyz
 jiyoshops.site
 jiyue.org
 jiyun123.com
@@ -55717,6 +55725,7 @@ justonetworks.site
 justpoleznoe.ru
 justpositiveweblog.club
 justrbonlinea.co.uk
+justre.codes
 justreadit.ru
 justruninabox.com
 justsend.online
@@ -66245,6 +66254,7 @@ mendingme.net
 mendoanmail.club
 mendycady.com
 menealacola.com
+menece.com
 menflo.com
 mengatur.com
 mengchong.info
@@ -66783,6 +66793,7 @@ mhtqq.icu
 mhwolf.net
 mhxcpxbna.shop
 mhzayt.com
+mhzayt.online
 mi-fucker-ss.ru
 mi-mails.com
 mi-turisto.ru
@@ -80872,6 +80883,7 @@ premier-reverse.com
 premierbetcasino.com
 premierbetsuper6.com
 premierclosing.com
+premiere.ee
 premiereatlantarealestate.com
 premierelivecasino.com
 premierloto.info
@@ -93326,6 +93338,7 @@ somsupport.xyz
 son-haber.live
 son-haber.site
 son-haber.website
+son16.com
 sonaluma.com
 sonamyr.shop
 sonasoft.net
@@ -104533,6 +104546,7 @@ uphonni.ru
 upimage.net
 upimail.com
 upived.com
+upived.online
 upizzatakeaway.com
 upl0adand1nsta11.com
 uplay-store.ru