email_data 1607321830 → 1607427861

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 89c4fcbee32f0a8df7950a9cb42b018d07727b5bc1856be40159e2bd9048a001
-  data.tar.gz: ddecb42320168c43e657af4766f11ef9dccad1ff36879046a48e73e6aaa661f5
+  metadata.gz: 63b2324c3c5b0b6cf82035d848fdc7bb4da8ddd7f4e5fd2ea5bacff09eb37fe0
+  data.tar.gz: 42505ee545597463a44604089b0d8288daee2a5b4bbddc237dbe9d6e20c59c0f
 SHA512:
-  metadata.gz: 1938455b425a6c6bccffe5fa4d54fd1f7560bbdd4bb174c670d5d62d1b1b7fbbca4e58e55dcfa7dc8811bc98051aec1ca5cc17bffc38ad6b06eb5a4efd3c5011
-  data.tar.gz: '08d6bf89db0ed7c42921c27c35f3618531c8f8125b2f3b100460495eb76d00f35b7701126862e37fbf3858f3d82b45dfefb00c880a5aa77f645dfac8bf4808a2'
+  metadata.gz: f46c3cc8c6760ca863e8d70804f48b6badf714368815cce42a8ce12c386849e79e1f8996cf4c1168cba143ca8360797a4463b75f8921416e6b7405f412cffe3b
+  data.tar.gz: dc79df9eecce6e95e44c15d806668ce585ee8147bc8ac94ea614e40e1992a3310d66a877ee95d3c8f67b933623532ea0b1ecc4651438386cf6da7e1d0ebc3881

data/.github/FUNDING.yml

@@ -1,4 +1,3 @@
-# These are supported funding model platforms
-
+---
 github: [fnando]
 custom: ["https://paypal.me/nandovieira/🍕"]

data/.github/workflows/package.yml

@@ -2,12 +2,10 @@
 name: package
 
 on:
-  push:
-    branches:
-      - publish
-
-  # schedule:
-  #   - cron: "0 10 * * *"
+  schedule:
+    - cron: "0 0 * * *"
+  workflow_dispatch:
+    inputs: {}
 
 jobs:
   build:
@@ -49,20 +47,3 @@ jobs:
           NPM_AUTH_TOKEN: "${{ secrets.NPM_AUTH_TOKEN }}"
         run: |
           ./bin/package
-          git config user.email "me@fnando.com"
-          git config user.name "Nando Vieira via Github Actions"
-          git add Gemfile.lock
-          git add package.json
-          git add lib/email_data/version.rb
-          git commit --message "Update lists."
-          git pull origin --rebase
-          git push origin --set-upstream
-
-          mkdir -p ~/.gem
-          echo ":rubygems_api_key: $RUBYGEMS_API_KEY" > ~/.gem/credentials
-          chmod 0600 ~/.gem/credentials
-          rake release
-
-          echo "//registry.npmjs.org/:_authToken=$NPM_AUTH_TOKEN" > ~/.npmrc
-          chmod 0600 ~/.npmrc
-          npm publish --access public

data/.gitignore

@@ -9,5 +9,24 @@
 /data/disposable/**/*
 /data/json/**/*
 LOCK
-/bin/jq
-/vendor
+bin/jq
+opt/
+vendor/
+pkg/
+VERSION
+.DS_Store
+.*.sw?
+/.pow*
+/.rvmrc
+/.rbenv-version
+/bin/
+/.tags*
+Thumbs.db
+.vagrant
+.bash/user.sh
+.gemtags
+.tags
+.tags_sorted_by_file
+/node_modules
+/.env
+/.env.*

data/README.md

@@ -41,6 +41,8 @@ EmailData.data_dir
 
 # List of disposable domains. Punycode is expanded into ASCII domains.
 EmailData.disposable_domains
+EmailData.disposable_domains_with_mx
+EmailData.disposable_domains_without_mx
 
 # List of disposable emails. Some services use free email like Gmail to create
 # disposable emails.
@@ -127,6 +129,18 @@ class SetupEmailData < ActiveRecord::Migration[6.1]
 
     add_index :disposable_domains, :name, unique: true
 
+    create_table :disposable_domains_with_mx do |t|
+      t.citext :name, null: false
+    end
+
+    add_index :disposable_domains_with_mx, :name, unique: true
+
+    create_table :disposable_domains_without_mx do |t|
+      t.citext :name, null: false
+    end
+
+    add_index :disposable_domains_without_mx, :name, unique: true
+
     create_table :free_email_domains do |t|
       t.citext :name, null: false
     end
@@ -176,6 +190,8 @@ COPY slds (name) FROM '/usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0/gems/email_data
 COPY country_tlds (name) FROM '/usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0/gems/email_data-1601479967/data/country_tlds.txt';
 COPY disposable_emails (name) FROM '/usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0/gems/email_data-1601479967/data/disposable_emails.txt';
 COPY disposable_domains (name) FROM '/usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0/gems/email_data-1601479967/data/disposable_domains.txt';
+COPY disposable_domains_with_mx (name) FROM '/usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0/gems/email_data-1601479967/data/disposable_domains_with_mx.txt';
+COPY disposable_domains_without_mx (name) FROM '/usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0/gems/email_data-1601479967/data/disposable_domains_without_mx.txt';
 COPY free_email_domains (name) FROM '/usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0/gems/email_data-1601479967/data/free_email_domains.txt';
 COPY roles (name) FROM '/usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0/gems/email_data-1601479967/data/roles.txt';
 COPY private_relays (name) FROM '/usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0/gems/email_data-1601479967/data/private_relays.txt';
@@ -204,6 +220,8 @@ class LoadEmailData < ActiveRecord::Migration[6.1]
     copy.call(:country_tlds)
     copy.call(:disposable_emails)
     copy.call(:disposable_domains)
+    copy.call(:disposable_domains_with_mx)
+    copy.call(:disposable_domains_without_mx)
     copy.call(:free_email_domains)
     copy.call(:roles)
     copy.call(:private_relays)
@@ -230,6 +248,8 @@ $ npm install @fnando/email_data
 ```js
 const disposableEmails = require("@fnando/email_data/data/json/disposable_emails.json");
 const disposableDomains = require("@fnando/email_data/data/json/disposable_domains.json");
+const disposableDomainsWithMx = require("@fnando/email_data/data/json/disposable_domains_with_mx.json");
+const disposableDomainsWithoutMx = require("@fnando/email_data/data/json/disposable_domains_without_mx.json");
 const freeEmailDomains = require("@fnando/email_data/data/json/free_email_domains.json");
 const roles = require("@fnando/email_data/data/json/roles.json");
 const privateRelays = require("@fnando/email_data/data/json/private_relays.json");

data/bin/helpers.rb

@@ -106,9 +106,9 @@ def random_timeout
   (1..10).map {|i| (1.0 / i).round(2) }.sample
 end
 
-def refresh_list(url:, path:, verb: :get, params: nil, &block)
+def refresh_list(url:, path:, verb: :get, params: nil)
   response = http_request(verb, url, params)
-  items = block.call(response)
+  items = yield(response)
   append_to_file(path, items)
 rescue StandardError => error
   puts "=> [ERROR] Unable to refresh list: #{url} (#{error.class}: #{error.message})"
@@ -191,9 +191,9 @@ def root_domains(domains)
   processed_domains.reject(&:nil?)
 end
 
-def thread(&block)
+def thread
   Thread.new do
-    block.call
+    yield
   rescue StandardError => error
     puts "=> [ERROR] Thread has errored; #{error.class}: #{error.message}"
   end

data/bin/package

@@ -8,11 +8,49 @@ chmod +x bin/jq
 which jq
 jq --version
 
-echo ./bin/sync
-./bin/export-json-files
+./bin/sync
 
 echo "=> Updating version files"
 version=$(date "+%s")
 package_json=$(cat package.json | jq --arg version "${version}.0.0" '.version = $version')
 echo $package_json | jq > package.json
 sed -i -E "s/[0-9]+/$version/" ./lib/email_data/version.rb
+
+git_status=$(git status 2> /dev/null | tr "\\n" " ")
+
+if [[ "$git_status" =~ "nothing to commit" ]]; then
+  echo "No changes detected. Skipping new packages."
+  exit
+fi
+
+git config user.email "me@fnando.com"
+git config user.name "Nando Vieira via Github Actions"
+git add Gemfile.lock
+git add package.json
+git add lib/email_data/version.rb
+git commit --message "Update lists."
+git reset --hard
+git pull origin --rebase
+git push origin --set-upstream
+git tag "v$version"
+git push --tags
+
+echo "=> Check repo status"
+git diff
+git diff-index --cached HEAD
+git status
+
+echo "=> Generate gem package"
+mkdir -p ~/.gem
+echo ":rubygems_api_key: $RUBYGEMS_API_KEY" > ~/.gem/credentials
+chmod 0600 ~/.gem/credentials
+mkdir -p pkg
+rm -rf pkg/*.gem
+email_data_gem=pkg/email_data-$version.gem
+gem build email_data.gemspec --silent --output $email_data_gem
+gem push $email_data_gem
+
+echo "=> Generate npm package"
+echo "//registry.npmjs.org/:_authToken=$NPM_AUTH_TOKEN" > ~/.npmrc
+chmod 0600 ~/.npmrc
+npm publish --access public --tag "v$version"

data/bin/sync-disposable-emails

@@ -130,7 +130,7 @@ def process_scraping(element, value_selector)
   else
     element.attributes.each do |_name, attr|
       attr = attr.value.to_s
-      value = attr if attr =~ host_regex
+      value = attr if attr&.match?(host_regex)
     end
   end
 
@@ -172,7 +172,6 @@ threads << thread { domain_scraping("itemp", "https://itemp.email", "select[name
 threads << thread { domain_scraping("cs", "https://www.cs.email", "select[id=gm-host-select] option::text()") }
 threads << thread { domain_scraping("tempmail", "https://tempmail.io/settings/", "select[id=domain] option::text()") }
 threads << thread { domain_scraping("tempemail", "https://tempemail.co", "select[name=email_domain] option::text()") }
-threads << thread { domain_scraping("tmail", "https://mytemp-email.com/", "a.domain-selector::text()") }
 
 threads.each_slice(5) do |slice|
   slice.each(&:join)

data/lib/email_data/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module EmailData
-  VERSION = "1607321830"
+  VERSION = "1607427861"
 end

data/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@fnando/email_data",
   "description": "This project is a compilation of datasets related to emails. Includes disposable emails, disposable domains, and free email services.",
-  "version": "1607321830.0.0",
+  "version": "1607427861.0.0",
   "license": "MIT",
   "repository": {
     "type": "git",

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: email_data
 version: !ruby/object:Gem::Version
-  version: '1607321830'
+  version: '1607427861'
 platform: ruby
 authors:
 - Nando Vieira
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-12-07 00:00:00.000000000 Z
+date: 2020-12-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -247,9 +247,9 @@ licenses:
 metadata:
   homepage_uri: https://github.com/fnando/email_data
   bug_tracker_uri: https://github.com/fnando/email_data/issues
-  source_code_uri: https://github.com/fnando/email_data/tree/v1607321830
-  documentation_uri: https://github.com/fnando/email_data/tree/v1607321830/README.md
-  license_uri: https://github.com/fnando/email_data/tree/v1607321830/LICENSE.md
+  source_code_uri: https://github.com/fnando/email_data/tree/v1607427861
+  documentation_uri: https://github.com/fnando/email_data/tree/v1607427861/README.md
+  license_uri: https://github.com/fnando/email_data/tree/v1607427861/LICENSE.md
 post_install_message: 
 rdoc_options: []
 require_paths: