validators 3.3.0 → 3.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 862ad7c2a49bcc38d917336b109047a330c6de6c6da16d83ea7953ddefe70096
-  data.tar.gz: fb5e0b9db3ec0fdbc064eb6467cb36fab3d0f4bab6f29a5c8439c8d5edeb46fd
+  metadata.gz: 49077fb99da3c1c92a3b981cdfd0354acb5a62a9b433512658bfc5c1e4df649b
+  data.tar.gz: 9d48b1cf25f3782afeaa00c0fa7f51ecf5581ea7e6bb49ebd5796ab914aecb50
 SHA512:
-  metadata.gz: b8842cf3ee2161ef2fe18c384efdb5ff6407c135560a95b91689d29b8ff65d87272c599288de55a6422dfaa6ff4be3b0b00fbd7ea8a8472e47bc08acfb973a85
-  data.tar.gz: 7a6fe664d6ef7a33735b8bf41cc198f8509a5712f788a2db93d6f2c4b39678ad6965eea1557fb6e43551a4d947a0cfd2e89c16a53c0721b6ada4390baa8cf293
+  metadata.gz: 1b307ad09f0e26c9e1db7766a15512aab5c87eb5cbd6217e9da489a45abce98bad30c4c320d0042d46fbf92d1d2c061487736f45d723d569a04cf26fb27bc453
+  data.tar.gz: b72f412e65628ac2d4b1e14160e2fcc855383ff3e40641d78e77cfa99f3f9b781e82d9cc4ca269ecd45493f4b199d918102b713d7860c99bf73e88b73ab6e060

data/.github/FUNDING.yml

@@ -0,0 +1 @@
+github: [fnando]

data/.rubocop.yml

@@ -4,6 +4,8 @@ inherit_gem:
 
 AllCops:
   TargetRubyVersion: 2.6
+  Exclude:
+    - vendor/**/*
 
 Style/AsciiComments:
   Enabled: false

data/Rakefile

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "bundler/setup"
 require "bundler/gem_tasks"
 require "rake/testtask"
 require "rubocop/rake_task"

data/lib/validators.rb

@@ -7,7 +7,7 @@ module Validators
   require "validators/ip"
   require "validators/tld"
   require "validators/hostname"
-  require "validators/disposable_hostnames"
+  require "validators/disposable_domains"
   require "validators/disposable_emails"
   require "validators/reserved_subdomains"
 

data/lib/validators/disposable_domains.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Validators
+  class DisposableDomains
+    def self.all
+      @all ||=
+        begin
+          require "email_data"
+          EmailData.disposable_domains
+        rescue LoadError
+          raise "email_data is not part of the bundle. Add it to Gemfile."
+        end
+    end
+
+    def self.include?(domain)
+      all.include?(domain)
+    end
+  end
+end

data/lib/validators/disposable_emails.rb

@@ -2,10 +2,14 @@
 
 module Validators
   class DisposableEmails
-    FILE_PATH = File.expand_path("../../data/disposable_emails.txt", __dir__)
-
     def self.all
-      @all ||= File.read(FILE_PATH).lines.map(&:chomp)
+      @all ||=
+        begin
+          require "email_data"
+          EmailData.disposable_emails
+        rescue LoadError
+          raise "email_data is not part of the bundle. Add it to Gemfile."
+        end
     end
 
     def self.include?(email)

data/lib/validators/tld.rb

@@ -2,10 +2,14 @@
 
 module Validators
   class TLD
-    FILE_PATH = File.expand_path("../../data/tld.txt", __dir__)
-
     def self.all
-      @all ||= File.read(FILE_PATH).lines.map(&:chomp)
+      @all ||=
+        begin
+          require "email_data"
+          EmailData.tlds
+        rescue LoadError
+          raise "email_data is not part of the bundle. Add it to Gemfile."
+        end
     end
 
     def self.host_with_valid_tld?(host)
@@ -13,10 +17,10 @@ module Validators
 
       return false if host.split(".").size == 1
 
-      valid?(host[/\.([^.]+)$/, 1].to_s.downcase)
+      include?(host[/\.([^.]+)$/, 1].to_s.downcase)
     end
 
-    def self.valid?(tld)
+    def self.include?(tld)
       all.include?(tld)
     end
   end

data/lib/validators/validates_email_format_of.rb

@@ -50,7 +50,7 @@ module ActiveModel
         hostname = value.to_s.split(AT_SIGN).last.to_s.downcase
         root_domain = RootDomain.call(hostname)
 
-        return unless Validators::DisposableHostnames.all.include?(root_domain)
+        return unless Validators::DisposableDomains.include?(root_domain)
 
         record.errors.add(
           attribute,

data/lib/validators/version.rb

@@ -3,7 +3,7 @@
 module Validators
   module Version
     MAJOR = 3
-    MINOR = 3
+    MINOR = 4
     PATCH = 0
     STRING = "#{MAJOR}.#{MINOR}.#{PATCH}"
   end

data/test/test_helper.rb

@@ -3,7 +3,6 @@
 $VERBOSE = nil
 
 require "simplecov"
-SimpleCov.start
 
 SimpleCov.start do
   add_filter "test/support"
@@ -31,7 +30,7 @@ end
 
 Time.zone = "America/Sao_Paulo"
 TLDs = Validators::TLD.all.sample(10)
-DISPOSABLE_DOMAINS = Validators::DisposableHostnames.all.sample(10)
+DISPOSABLE_DOMAINS = Validators::DisposableDomains.all.sample(10)
 DISPOSABLE_EMAILS = Validators::DisposableEmails.all +
                     Validators::DisposableEmails.all.sample(10).map {|email| build_email_with_filter(email) } +
                     Validators::DisposableEmails.all.sample(10).map {|email| build_email_with_dots(email) } +

data/test/validators/validates_url_format_of/with_tld_validation_test.rb

@@ -14,4 +14,22 @@ class ValidatesurlFormatUrlWithTldValidationTest < Minitest::Test
       assert user.valid?
     end
   end
+
+  test "rejects invalid TLD (alternative syntax)" do
+    user_model = Class.new do
+      include ActiveModel::Validations
+      attr_accessor :site_url
+
+      def self.name
+        "User"
+      end
+
+      validates :site_url, url: {tld: true}
+    end
+
+    user = user_model.new
+    user.site_url = "https://example.xy"
+
+    refute user.valid?
+  end
 end

data/test/validators/validates_url_format_of/without_tld_validation_test.rb

@@ -47,4 +47,22 @@ class ValidatesurlFormatUrlWithoutTldValidationTest < Minitest::Test
     user = User.new(url: "")
     refute user.valid?
   end
+
+  test "accepts invalid TLD (alternative syntax)" do
+    user_model = Class.new do
+      include ActiveModel::Validations
+      attr_accessor :site_url
+
+      def self.name
+        "User"
+      end
+
+      validates :site_url, url: {tld: false}
+    end
+
+    user = user_model.new
+    user.site_url = "https://example.xy"
+
+    assert user.valid?
+  end
 end

data/validators.gemspec

@@ -12,6 +12,7 @@ Gem::Specification.new do |s|
   s.summary     = "Add some nice ActiveModel/ActiveRecord validators."
   s.description = s.summary
   s.license     = "MIT"
+  s.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
@@ -23,6 +24,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency "activerecord"
   s.add_development_dependency "aitch"
   s.add_development_dependency "cpf_cnpj"
+  s.add_development_dependency "email_data"
   s.add_development_dependency "minitest-utils"
   s.add_development_dependency "mocha"
   s.add_development_dependency "nokogiri"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: validators
 version: !ruby/object:Gem::Version
-  version: 3.3.0
+  version: 3.4.0
 platform: ruby
 authors:
 - Nando Vieira
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-22 00:00:00.000000000 Z
+date: 2020-09-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: email_data
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: minitest-utils
   requirement: !ruby/object:Gem::Requirement
@@ -265,29 +279,22 @@ dependencies:
 description: Add some nice ActiveModel/ActiveRecord validators.
 email:
 - fnando.vieira@gmail.com
-executables:
-- sync-disposable-hostnames
-- sync-tld
+executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/FUNDING.yml"
 - ".gitignore"
 - ".rubocop.yml"
 - ".travis.yml"
 - Gemfile
 - README.md
 - Rakefile
-- bin/sync-disposable-hostnames
-- bin/sync-tld
-- data/country_tlds.txt
-- data/disposable_domains.txt
-- data/disposable_emails.txt
 - data/reserved_subdomains.txt
-- data/tld.txt
 - lib/validators.rb
 - lib/validators/constants.rb
+- lib/validators/disposable_domains.rb
 - lib/validators/disposable_emails.rb
-- lib/validators/disposable_hostnames.rb
 - lib/validators/hostname.rb
 - lib/validators/ip.rb
 - lib/validators/locale/en.yml
@@ -351,7 +358,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="

data/bin/sync-disposable-hostnames

@@ -1,230 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require_relative "helpers"
-
-def ten_minute_mail
-  path = "disposable/10minutemail.txt"
-  url = "https://10minutemail.com/session/address"
-
-  20.times do
-    refresh_list(url: url, path: path) do |response|
-      _account, host = response.data.fetch("address").split("@")
-
-      [host]
-    end
-
-    sleep random_timeout
-  end
-end
-
-def temp_mail
-  path = "disposable/tempmail.txt"
-  url = "https://api4.temp-mail.org/request/domains/format/json"
-
-  refresh_list(url: url, path: path) do |response|
-    response.data.map {|domain| domain.tr("@", "") }
-  end
-end
-
-def temp_mail_address
-  path = "disposable/tempmailaddress.txt"
-  url = "https://www.tempmailaddress.com/index/index"
-
-  refresh_list(url: url, path: path) do |response|
-    data = JSON.parse(
-      response.body.gsub(/[^-,:\w@.{}"]/, ""),
-      symbolize_names: true
-    )
-    [data[:email].split("@").last]
-  end
-end
-
-def tempmail_io
-  path = "disposable/tempmail_io.txt"
-  url = "https://api.internal.temp-mail.io/api/v2/domains"
-
-  refresh_list(url: url, path: path) do |response|
-    response.data["domains"]
-  end
-end
-
-def gmailnator
-  emails = []
-
-  5.times do
-    url = "https://gmailnator.com/bulk-emails"
-    default_headers = {"user-agent" => USER_AGENT.sample}
-
-    response = Aitch.get(url: url, headers: default_headers)
-
-    throw "Received #{response.status} when getting CSRF token" unless response.ok?
-
-    cookie_header = response.headers["set-cookie"]
-    attr = response.data.css("#csrf_token").first
-    csrf_token = attr[:value]
-    csrf_field = attr[:name]
-
-    response = Aitch.post(
-      url: url,
-      params: {email_list: "1000", email: [3], csrf_field => csrf_token},
-      headers: default_headers.merge({"cookie" => cookie_header})
-    )
-
-    throw "Received #{response.status} when fetching list" unless response.ok?
-
-    emails += response.data.css("#email-list-message a").map do |node|
-      mailbox, domain = node.text.gsub(/\+[^@]+/, "").split("@")
-      mailbox = mailbox.gsub(/\./m, "")
-      "#{mailbox}@#{domain}"
-    end
-
-    sleep random_timeout
-  end
-
-  append_to_file("disposable/gmailnator.txt", emails)
-end
-
-def domain_scraping(name, url, selector)
-  timeout(10) do
-    puts "=> Scraping #{url}"
-
-    selector, value_selector = selector.split("::")
-    path = "disposable/#{name}.txt"
-    host_regex = /@?(.*?(\.[^.]+)+)/
-
-    refresh_list(url: url, path: path) do |response|
-      new_domains = response
-                    .data
-                    .css(selector)
-                    .map {|element| process_scraping(element, value_selector) }
-
-      new_domains = new_domains
-                    .map(&:squish)
-                    .reject(&:empty?)
-                    .map {|domain| domain[host_regex, 1]&.squish&.tr("@", "") }
-                    .reject(&:nil?)
-                    .reject(&:empty?)
-                    .map {|domain| domain.gsub(/\s*\((.*?)\)/, "") }
-
-      raise "No #{name} hosts found" if new_domains.empty?
-
-      new_domains
-    end
-  end
-rescue StandardError => error
-  puts "=> [ERROR] Unable to scrape #{url}; #{error.class}: #{error.message}"
-  []
-end
-
-def process_scraping(element, value_selector)
-  value = nil
-
-  case value_selector
-  when "text()"
-    value = element.text
-  when /^attr\((.*?)\)/
-    value = element[Regexp.last_match(1)]
-  else
-    element.attributes.each do |_name, attr|
-      attr = attr.value.to_s
-      value = attr if attr =~ host_regex
-    end
-  end
-
-  raise "no value found: #{element} (value_selector: #{value_selector})" unless value
-
-  value
-end
-
-def load_github_url(url)
-  puts "=> Fetching #{url}"
-
-  basename = URI.parse(url).path[%r{/([^/]+/[^/]+)}, 1].tr("/", "_").tr("-", "_")
-  path = "disposable/#{basename}.txt"
-  domains = load_file(path)
-
-  ext = File.extname(url)
-
-  domains += case ext
-             when ".json"
-               JSON.parse(http_request(:get, url).body)
-             when ".txt"
-               http_request(:get, url).body.lines.map(&:chomp)
-             else
-               raise "Unknown extension"
-             end
-
-  append_to_file(path, domains)
-  domains
-rescue StandardError => error
-  puts "=> Unable to load #{url}; #{error.class}: #{error.message}"
-  []
-end
-
-threads = []
-
-threads << thread { load_github_url("https://raw.githubusercontent.com/ivolo/disposable-email-domains/master/index.json") }
-threads << thread { load_github_url("https://raw.githubusercontent.com/andreis/disposable-email-domains/master/domains.json") }
-threads << thread { load_github_url("https://raw.githubusercontent.com/FGRibreau/mailchecker/master/list.txt") }
-threads << thread { load_github_url("https://raw.githubusercontent.com/willwhite/freemail/master/data/disposable.txt") }
-threads << thread { load_github_url("https://raw.githubusercontent.com/maxmalysh/disposable-emails/master/disposable_emails/data/domains.txt") }
-threads << thread { load_github_url("https://raw.githubusercontent.com/jespernissen/disposable-maildomain-list/master/disposable-maildomain-list.txt") }
-threads << thread { load_github_url("https://raw.githubusercontent.com/wesbos/burner-email-providers/master/emails.txt") }
-threads << thread { load_github_url("https://gist.github.com/fnando/dafe542cac13f831bbf5521a55248116/raw/disposable.txt") }
-threads << thread { ten_minute_mail }
-threads << thread { temp_mail }
-threads << thread { temp_mail_address }
-threads << thread { tempmail_io }
-threads << thread { load_file("disposable/disposable_manually_added.txt") }
-threads << thread { domain_scraping("guerrillamail", "https://www.guerrillamail.com/", "select option::attr(value)") }
-threads << thread { domain_scraping("moakt", "https://www.moakt.com", "select option::attr(value)") }
-threads << thread { domain_scraping("tempr", "https://tempr.email/", "select[name=DomainId] option::text()") }
-threads << thread { domain_scraping("yepmail", "https://yepmail.co/", "select[name=domain] option::text()") }
-threads << thread { domain_scraping("fake_email_generator", "https://fakemailgenerator.net", "[data-mailhost]::attr(data-mailhost)") }
-threads << thread { domain_scraping("tempemails", "https://www.tempemails.net/", "select[name=domain] option::attr(value)") }
-threads << thread { domain_scraping("clipmails", "https://clipmails.com/", "select[name=domain] option::attr(value)") }
-threads << thread { domain_scraping("1secmail", "https://www.1secmail.com/", "select[id=domain] option::attr(value)") }
-threads << thread { domain_scraping("emailfake", "https://generator.email", ".tt-suggestion p::text()") }
-threads << thread { domain_scraping("emailfake", "https://emailfake.com/", ".tt-suggestion p::text()") }
-threads << thread { domain_scraping("emailfake", "https://email-fake.com/", ".tt-suggestion p::text()") }
-threads << thread { domain_scraping("receivemail", "https://www.receivemail.org/", "select[name=domain] option::text()") }
-threads << thread { domain_scraping("itemp", "https://itemp.email", "select[name=domain] option::text()") }
-threads << thread { domain_scraping("cs", "https://www.cs.email", "select[id=gm-host-select] option::text()") }
-threads << thread { domain_scraping("tempmail", "https://tempmail.io/settings/", "select[id=domain] option::text()") }
-threads << thread { domain_scraping("tempemail", "https://tempemail.co", "select[name=email_domain] option::text()") }
-threads << thread { domain_scraping("tmail", "https://mytemp-email.com/", "a.domain-selector::text()") }
-
-threads.each_slice(5) do |slice|
-  slice.each(&:join)
-end
-
-threads.clear
-
-domains = []
-
-puts "=> Loading disposable_domains.txt"
-domains += File.read("#{__dir__}/../data/disposable_domains.txt").lines.map(&:chomp)
-
-puts "=> Loading disposable/*.txt"
-Dir["./data/disposable/**/*.txt"].map do |file|
-  file = File.expand_path(file)
-  domains += File.read(file).lines.map(&:chomp).flatten.compact
-end
-
-ignore_domains = %w[gmail.com hotmail.com]
-
-puts "=> Normalize domains (count: #{domains.size})"
-domains = domains
-          .uniq
-          .map {|domain| RootDomain.call(domain.split("@").last.downcase) }
-          .compact
-          .uniq
-          .reject {|domain| ignore_domains.include?(domain) }
-
-puts "=> Saving domains (count: #{domains.size})"
-save_file("disposable_domains.txt", domains)
-
-emails = gmailnator
-puts "=> Saving email proxies (count: #{emails.size})"
-save_file("disposable_emails.txt", emails)