email_data 1601260789 → 1602964602

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7d31b668bb75b28facd8b15d760c381066044793a21182bcc1a425769702b84d
-  data.tar.gz: 2e3753a694119b5b954f7a4e0b3a40d4dddca956f52acca22708a48fcfb5b8d7
+  metadata.gz: 68a57adbf3234a937d344be152e4de4473d279de1d2a9afc5039cb7e49b14ea0
+  data.tar.gz: 5f5cba8455f02f644ebb66c68621cd51c14cda3fbbcc9930526f31f74352a5b2
 SHA512:
-  metadata.gz: '02923753d75d4ebbd5e56c0d29cd9f57a397c7e118ab0caec6a36440a7cb2dc4d33a5f9d9138869b47ed5d02a929d52ae6096f778300854ed270459beffb09b3'
-  data.tar.gz: 60559acd391158f31439fe843c87990e4c5e765e5cef9b24c1ec88d6bcae654bed04b1d2f7410e8839c429b4f7c77f4f09037cb296b4137078bb22836d793a54
+  metadata.gz: fba343c31cca846cabfb19cf009bf1664a9cc30ee45389e9620bc2486fd82a3d3bb70fe0cea4de056a79ff80cb9c7e9f7039809ff4a9c9666f72c8241eb0d556
+  data.tar.gz: 546e18b43810c3e3f194ee855af176bf480bf4100c1dc4d6e12cb3d3ae7f634b97e90cc3073e9fcacd67c4212bfae2dca73633680ac435c12ac4829538350bc6

data/.github/workflows/tests.yml

@@ -0,0 +1,67 @@
+name: Tests
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    name: Tests with Ruby ${{ matrix.ruby }}
+    runs-on: "ubuntu-latest"
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: ["2.7.x", "2.6.x", "2.5.x"]
+
+    services:
+      postgres:
+        image: postgres:11.5
+        ports: ["5432:5432"]
+        options:
+          --health-cmd pg_isready --health-interval 10s --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - uses: actions/checkout@v1
+
+      - uses: actions/cache@v2
+        with:
+          path: vendor/bundle
+          key: >
+            ${{ runner.os }}-${{ matrix.ruby }}-gems-${{
+            hashFiles('**/Gemfile.lock') }}
+          restore-keys: >
+            ${{ runner.os }}-${{ matrix.ruby }}-gems-${{
+            hashFiles('**/Gemfile.lock') }}
+
+      - name: Set up Ruby
+        uses: actions/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+
+      - name: Install PostgreSQL 11 client
+        run: |
+          sudo apt-get -yqq install libpq-dev
+
+      - name: Install gem dependencies
+        env:
+          PGHOST: localhost
+          PGUSER: postgres
+          RAILS_ENV: test
+        run: |
+          gem install bundler
+          bundle config path vendor/bundle
+          bundle install --jobs 4 --retry 3
+
+      - name: Run Tests
+        env:
+          PGHOST: localhost
+          PGUSER: postgres
+          RAILS_ENV: test
+        run: |
+          psql -U postgres -c "create database test"
+          bundle exec rake

data/.gitignore

@@ -8,4 +8,4 @@
 /tmp/
 /data/disposable/**/*
 /data/json/**/*
-Gemfile.lock
+LOCK

data/.rubocop.yml

@@ -3,5 +3,18 @@ inherit_gem:
   rubocop-fnando: .rubocop.yml
 
 AllCops:
-  TargetRubyVersion: 2.7
+  TargetRubyVersion: 2.5
   NewCops: enable
+
+Metrics/AbcSize:
+  Exclude:
+    - test/support/**/*
+
+Metrics/MethodLength:
+  Exclude:
+    - test/support/**/*
+
+Metrics/BlockLength:
+  Exclude:
+    - email_data.gemspec
+    - test/support/**/*

data/.tool-versions

@@ -1,2 +1,2 @@
-ruby 2.7.1
+ruby 2.7.2
 nodejs 12.18.4

data/Gemfile.lock

@@ -0,0 +1,89 @@
+PATH
+  remote: .
+  specs:
+    email_data (1602964602)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (6.0.3.4)
+      activesupport (= 6.0.3.4)
+    activerecord (6.0.3.4)
+      activemodel (= 6.0.3.4)
+      activesupport (= 6.0.3.4)
+    activesupport (6.0.3.4)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+      zeitwerk (~> 2.2, >= 2.2.2)
+    aitch (1.1.0)
+      nokogiri
+    ast (2.4.1)
+    concurrent-ruby (1.1.7)
+    docile (1.3.2)
+    i18n (1.8.5)
+      concurrent-ruby (~> 1.0)
+    mini_portile2 (2.4.0)
+    minitest (5.14.2)
+    minitest-utils (0.4.6)
+      minitest
+    nokogiri (1.10.10)
+      mini_portile2 (~> 2.4.0)
+    parallel (1.19.2)
+    parser (2.7.2.0)
+      ast (~> 2.4.1)
+    pg (1.2.3)
+    rainbow (3.0.0)
+    rake (13.0.1)
+    regexp_parser (1.8.2)
+    rexml (3.2.4)
+    root_domain (0.1.0)
+      simpleidn
+    rubocop (0.93.1)
+      parallel (~> 1.10)
+      parser (>= 2.7.1.5)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8)
+      rexml
+      rubocop-ast (>= 0.6.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 2.0)
+    rubocop-ast (0.8.0)
+      parser (>= 2.7.1.5)
+    rubocop-fnando (0.0.13)
+    ruby-progressbar (1.10.1)
+    simplecov (0.19.0)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+    simplecov-html (0.12.3)
+    simpleidn (0.1.1)
+      unf (~> 0.1.4)
+    thread_safe (0.3.6)
+    tzinfo (1.2.7)
+      thread_safe (~> 0.1)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.7)
+    unicode-display_width (1.7.0)
+    zeitwerk (2.4.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activerecord
+  aitch
+  bundler
+  email_data!
+  minitest
+  minitest-utils
+  pg
+  rake
+  root_domain
+  rubocop
+  rubocop-fnando
+  simplecov
+
+BUNDLED WITH
+   2.1.4

data/README.md

@@ -1,5 +1,15 @@
 # EmailData
 
+- Ruby:
+  [![Tests](https://github.com/fnando/email_data/workflows/Tests/badge.svg)](https://github.com/fnando/email_data/actions?query=workflow%3ATests)
+  [![Gem](https://img.shields.io/gem/v/email_data.svg)](https://rubygems.org/gems/email_data)
+  [![Gem](https://img.shields.io/gem/dt/email_data.svg?label=gems%20downloads)](https://rubygems.org/gems/email_data)
+- NPM:
+  [![NPM package version](https://img.shields.io/npm/v/@fnando/email_data.svg)](https://www.npmjs.com/package/@fnando/email_data)
+  [![NPM Downloads](https://img.shields.io/npm/dt/@fnando/email_data?label=npm%20downloads)](https://www.npmjs.com/package/@fnando/email_data)
+- License:
+  ![License](https://img.shields.io/static/v1?label=License&message=MIT&color=4da3dd)
+
 This project is a compilation of datasets related to emails.
 
 - Disposable emails
@@ -46,6 +56,128 @@ EmailData.disposable_emails
 EmailData.free_email_domains
 ```
 
+#### Data sources
+
+By default, Ruby will load data from filesystem. You may want to load this data
+from the database instead. `email-data` has support for ActiveRecord out of the
+box. To use the ActiveRecord adapter, you must load
+`email_data/source/active_record.rb`. You can easily do it so with Bundler's
+`require` key.
+
+```ruby
+gem "email_data", require: "email_data/source/active_record"
+```
+
+Then, you need to assign the new data source.
+
+```ruby
+EmailData.source = EmailData::Source::ActiveRecord
+```
+
+If you need to configure a different database connection than the one defined by
+`ActiveRecord::Base`, use `EmailData::Source::ActiveRecord::ApplicationRecord`
+for that.
+
+##### Creating the tables
+
+To create the tables, use the migration code below (tweak it accordingly if you
+use something different than PostgreSQL, or don't want to use `citext`).
+
+```ruby
+class SetupEmailData < ActiveRecord::Migration[6.1]
+  def change
+    enable_extension "citext"
+
+    create_table :tlds do |t|
+      t.citext :name, null: false
+    end
+
+    add_index :tlds, :name, unique: true
+
+    create_table :country_tlds do |t|
+      t.citext :name, null: false
+    end
+
+    add_index :country_tlds, :name, unique: true
+
+    create_table :disposable_emails do |t|
+      t.citext :name, null: false
+    end
+
+    add_index :disposable_emails, :name, unique: true
+
+    create_table :disposable_domains do |t|
+      t.citext :name, null: false
+    end
+
+    add_index :disposable_domains, :name, unique: true
+
+    create_table :free_email_domains do |t|
+      t.citext :name, null: false
+    end
+
+    add_index :free_email_domains, :name, unique: true
+  end
+end
+```
+
+##### Loading the data
+
+With PostgreSQL, you load the data using the `COPY` command. First, you'll need
+to discover where your gems are being installed. Use `gem list` for that.
+
+```console
+$ gem list email_data -d
+
+*** LOCAL GEMS ***
+
+email_data (1601479967, 1601260789)
+    Author: Nando Vieira
+    Homepage: https://github.com/fnando/email_data
+    License: MIT
+    Installed at (1601479967): /usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0
+    This project is a compilation of datasets related to emails.
+    Includes disposable emails, disposable domains, and free email
+    services.
+```
+
+The you can load each dataset using `COPY`:
+
+```sql
+COPY tlds (name) FROM '/usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0/gems/email_data-1601479967/data/tlds.txt';
+COPY country_tlds (name) FROM '/usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0/gems/email_data-1601479967/data/country_tlds.txt';
+COPY disposable_emails (name) FROM '/usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0/gems/email_data-1601479967/data/disposable_emails.txt';
+COPY disposable_domains (name) FROM '/usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0/gems/email_data-1601479967/data/disposable_domains.txt';
+COPY free_email_domains (name) FROM '/usr/local/ruby/2.7.1/lib/ruby/gems/2.7.0/gems/email_data-1601479967/data/free_email_domains.txt';
+```
+
+Alternatively, you could create a migrate that executes that same command; given
+that you'd be running Ruby code, you can replace the steps to find the gem path
+with `EmailData.data_dir`.
+
+```ruby
+class LoadEmailData < ActiveRecord::Migration[6.1]
+  def change
+    copy = lambda do |table_name|
+      connection = ActiveRecord::Base.connection
+      data_path = EmailData.data_dir
+
+      connection.execute <<~PG
+        COPY #{table_name} (name)
+        FROM '#{data_path.join(table_name)}.txt'
+        (FORMAT CSV)
+      PG
+    end
+
+    copy.call(:tlds)
+    copy.call(:country_tlds)
+    copy.call(:disposable_emails)
+    copy.call(:disposable_domains)
+    copy.call(:free_email_domains)
+  end
+end
+```
+
 ### Node.js
 
 ```console

data/Rakefile

@@ -2,6 +2,7 @@
 
 require "bundler/gem_tasks"
 require "rake/testtask"
+require "rubocop/rake_task"
 
 Rake::TestTask.new(:test) do |t|
   t.libs << "test"
@@ -9,4 +10,6 @@ Rake::TestTask.new(:test) do |t|
   t.test_files = FileList["test/**/*_test.rb"]
 end
 
-task default: :test
+RuboCop::RakeTask.new
+
+task default: %i[test rubocop]

data/VERSION

@@ -1 +1 @@
-1601260789
+1602964602

data/bin/helpers.rb

@@ -6,6 +6,7 @@ Encoding.default_external = Encoding.find("UTF-8")
 Thread.abort_on_exception = false
 Thread.report_on_exception = false
 
+require "bundler/setup"
 require "open-uri"
 require "json"
 require "aitch"
@@ -14,6 +15,7 @@ require "simpleidn"
 require "timeout"
 require "active_support/all"
 require "root_domain"
+require "email_data"
 
 USER_AGENT = [
   "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.4 Safari/605.1.15",

data/bin/publish

@@ -5,6 +5,20 @@ set -e
 source $HOME/.zsh/exports.sh
 source $HOME/.zsh/asdf.sh
 
+lockfile=$(dirname $0)/../LOCK
+
+if [[ -f "$lockfile" ]]; then
+  terminal-notifier \
+    -message "Skipping script; repository is locked." \
+    -ignoreDnD \
+    -title "email_data" \
+    -group homebrew \
+    &> /dev/null
+  exit 1
+fi
+
+git reset --hard
+git checkout main
 git pull --rebase
 git push
 
@@ -19,7 +33,7 @@ if [[ "$git_status" =~ "nothing to commit" ]]; then
     -title "email_data" \
     -group homebrew \
     &> /dev/null
-  exit
+  exit 1
 fi
 
 date "+%s" > VERSION
@@ -29,6 +43,11 @@ package_json=$(cat package.json | jq --arg version "${version}.0.0" '.version =
 
 echo $package_json | jq --tab > package.json
 
+asdf local ruby 2.7.2
+
+gem install bundler
+bundle update
+
 git add .
 git commit -m "Bump up version."
 git push

data/bin/publish-gem

@@ -5,16 +5,12 @@ set -e
 source $HOME/.zsh/exports.sh
 source $HOME/.zsh/asdf.sh
 
-asdf local ruby 2.7.1
-
-gem install bundler
-bundle update
-
+asdf local ruby 2.7.2
 rake release
 
 terminal-notifier \
   -message "Just published a new gem version." \
   -ignoreDnD \
   -title "email_data" \
-  -group homebrew \
+  -group email_data_gem \
   &> /dev/null

data/bin/publish-npm

@@ -18,6 +18,6 @@ npm publish --access=public
 terminal-notifier \
   -message "Just published a new npm version." \
   -ignoreDnD \
-  -title "email_data" \
+  -title "email_data_npm" \
   -group homebrew \
   &> /dev/null