validators 3.2.1 → 3.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6e2c068d28878c6cd22e62c6842ed48300e66edf545693b97e08fb6e8ed01d53
-  data.tar.gz: 95b90e034d61f65ceb1a1538f906266f882b35d2ca8438b5186ea5caba2f9b6c
+  metadata.gz: 862ad7c2a49bcc38d917336b109047a330c6de6c6da16d83ea7953ddefe70096
+  data.tar.gz: fb5e0b9db3ec0fdbc064eb6467cb36fab3d0f4bab6f29a5c8439c8d5edeb46fd
 SHA512:
-  metadata.gz: d85e1a89b5b106d7b1e69809f7a314f001ada7e68416a74d8a08a3875bacd55c6634d02792ac53fe418c63a41253ea3787f2ff5bc257309c01c1cdbb07c07bcf
-  data.tar.gz: a12a716b4960cf40fd37f23ff0d049fe2c5efca3dc310c65c96e24d0a5c714c54c7426cb8170ce7ba45023d6367714833e61462db5c97bc6751774b946194d3e
+  metadata.gz: b8842cf3ee2161ef2fe18c384efdb5ff6407c135560a95b91689d29b8ff65d87272c599288de55a6422dfaa6ff4be3b0b00fbd7ea8a8472e47bc08acfb973a85
+  data.tar.gz: 7a6fe664d6ef7a33735b8bf41cc198f8509a5712f788a2db93d6f2c4b39678ad6965eea1557fb6e43551a4d947a0cfd2e89c16a53c0721b6ada4390baa8cf293

data/.gitignore

@@ -3,3 +3,4 @@ pkg/*
 .bundle
 Gemfile.lock
 /coverage
+/data/disposable/*

data/.rubocop.yml

@@ -33,3 +33,6 @@ Metrics/BlockLength:
   Exclude:
     - bin/**/*
     - "*.gemspec"
+
+Layout/EmptyLinesAroundAttributeAccessor:
+  Enabled: false

data/README.md

@@ -32,8 +32,7 @@ class User < ActiveRecord::Base
 end
 ```
 
-By default, it rejects disposable e-mails (e.g. mailinator). This loads ~15kb,
-but you can disable this validation by setting `disposable: true`.
+By default, it rejects disposable e-mails (e.g. mailinator). This loads a lot of data (~1.7MB), but you can disable this validation by setting `disposable: true`.
 
 ```ruby
 class User < ActiveRecord::Base
@@ -56,7 +55,7 @@ class User < ActiveRecord::Base
   validates_url_format_of :site
 
   # validates TLD against list of valid TLD.
-  # Loads ~5kb of text.
+  # Loads ~10KB of text.
   validates_url_format_of :site, tld: true
 end
 ```
@@ -153,7 +152,8 @@ A valid username/subdomain follows the hostname label validation:
 - cannot begin or end with a hyphen
 - cannot consist of numeric values only
 
-The compiled list will be used for both username and subdomain validations.
+The compiled list will be used for both username and subdomain validations. 
+This validation loads ~20KB of text.
 
 ```ruby
 class Server < ActiveRecord::Base

data/bin/sync-disposable-hostnames

@@ -4,18 +4,22 @@
 require_relative "helpers"
 
 def ten_minute_mail
-  path = "disposable/10minutemail.json"
-  url = "https://10minutemail.com/10MinuteMail/resources/session/address"
+  path = "disposable/10minutemail.txt"
+  url = "https://10minutemail.com/session/address"
 
-  refresh_list(url: url, path: path) do |response|
-    _account, host = response.body.split("@")
+  20.times do
+    refresh_list(url: url, path: path) do |response|
+      _account, host = response.data.fetch("address").split("@")
+
+      [host]
+    end
 
-    [host]
+    sleep random_timeout
   end
 end
 
 def temp_mail
-  path = "disposable/tempmail.json"
+  path = "disposable/tempmail.txt"
   url = "https://api4.temp-mail.org/request/domains/format/json"
 
   refresh_list(url: url, path: path) do |response|
@@ -24,7 +28,7 @@ def temp_mail
 end
 
 def temp_mail_address
-  path = "disposable/tempmailaddress.json"
+  path = "disposable/tempmailaddress.txt"
   url = "https://www.tempmailaddress.com/index/index"
 
   refresh_list(url: url, path: path) do |response|
@@ -37,7 +41,7 @@ def temp_mail_address
 end
 
 def tempmail_io
-  path = "disposable/tempmail_io.json"
+  path = "disposable/tempmail_io.txt"
   url = "https://api.internal.temp-mail.io/api/v2/domains"
 
   refresh_list(url: url, path: path) do |response|
@@ -46,69 +50,67 @@ def tempmail_io
 end
 
 def gmailnator
-  path = "disposable/gmailnator.json"
-  url = "https://gmailnator.com/index/indexquery"
-
-  refresh_list(
-    verb: :post,
-    url: url,
-    path: path,
-    params: {action: "GenerateEmail"}
-  ) do |response|
-    email = response.body.gsub(/(\+[^@]+)/, "")
-    [email]
-  end
-end
+  emails = []
 
-def prepare_patterns(domains)
-  new_domains = domains.map do |domain|
-    {
-      domain: domain,
-      processed: domain.gsub(/\..*?$/, "")
-    }
-  end
+  5.times do
+    url = "https://gmailnator.com/bulk-emails"
+    default_headers = {"user-agent" => USER_AGENT.sample}
 
-  stats = new_domains
-          .count_by {|info| info[:processed] }
-          .select {|_processed, count| count > 2 }
+    response = Aitch.get(url: url, headers: default_headers)
 
-  stats_map = stats.each_with_object({}) do |(name, count), buffer|
-    buffer[name] = count
-  end
+    throw "Received #{response.status} when getting CSRF token" unless response.ok?
 
-  domains = new_domains
-            .reject {|info| stats_map[info[:processed]] }
-            .map {|info| info[:domain] }
-  domains += stats_map.keys.map {|processed| "/#{processed}\\..+$/" }
+    cookie_header = response.headers["set-cookie"]
+    attr = response.data.css("#csrf_token").first
+    csrf_token = attr[:value]
+    csrf_field = attr[:name]
 
-  domains.select {|domain| domain.start_with?("/") }
-end
+    response = Aitch.post(
+      url: url,
+      params: {email_list: "1000", email: [3], csrf_field => csrf_token},
+      headers: default_headers.merge({"cookie" => cookie_header})
+    )
 
-def domain_scraping(name, url, selector)
-  puts "=> scraping #{url}"
+    throw "Received #{response.status} when fetching list" unless response.ok?
 
-  selector, value_selector = selector.split("::")
-  path = "disposable/#{name}.json"
-  host_regex = /@?(.*?(\.[^.]+)+)/
+    emails += response.data.css("#email-list-message a").map do |node|
+      mailbox, domain = node.text.gsub(/\+[^@]+/, "").split("@")
+      mailbox = mailbox.gsub(/\./m, "")
+      "#{mailbox}@#{domain}"
+    end
 
-  refresh_list(url: url, path: path) do |response|
-    new_domains = response
-                  .data
-                  .css(selector)
-                  .map {|element| process_scraping(element, value_selector) }
-
-    new_domains = new_domains
-                  .map(&:squeeze)
-                  .map(&:strip)
-                  .reject(&:empty?)
-                  .map {|domain| domain[host_regex, 1]&.squeeze&.tr("@", "") }
-                  .reject(&:nil?)
-                  .reject(&:empty?)
-                  .map {|domain| domain.gsub(/\s*\((.*?)\)/, "") }
-
-    raise "No #{name} hosts found" if new_domains.empty?
-
-    new_domains
+    sleep random_timeout
+  end
+
+  append_to_file("disposable/gmailnator.txt", emails)
+end
+
+def domain_scraping(name, url, selector)
+  timeout(10) do
+    puts "=> Scraping #{url}"
+
+    selector, value_selector = selector.split("::")
+    path = "disposable/#{name}.txt"
+    host_regex = /@?(.*?(\.[^.]+)+)/
+
+    refresh_list(url: url, path: path) do |response|
+      new_domains = response
+                    .data
+                    .css(selector)
+                    .map {|element| process_scraping(element, value_selector) }
+
+      new_domains = new_domains
+                    .map(&:squish)
+                    .reject(&:empty?)
+                    .map {|domain| domain[host_regex, 1]&.squish&.tr("@", "") }
+                    .reject(&:nil?)
+                    .reject(&:empty?)
+                    .map {|domain| domain.gsub(/\s*\((.*?)\)/, "") }
+
+      raise "No #{name} hosts found" if new_domains.empty?
+
+      new_domains
+    end
   end
 rescue StandardError => error
   puts "=> [ERROR] Unable to scrape #{url}; #{error.class}: #{error.message}"
@@ -139,7 +141,7 @@ def load_github_url(url)
   puts "=> Fetching #{url}"
 
   basename = URI.parse(url).path[%r{/([^/]+/[^/]+)}, 1].tr("/", "_").tr("-", "_")
-  path = "disposable/#{basename}.json"
+  path = "disposable/#{basename}.txt"
   domains = load_file(path)
 
   ext = File.extname(url)
@@ -155,48 +157,43 @@ def load_github_url(url)
 
   append_to_file(path, domains)
   domains
-rescue error
+rescue StandardError => error
   puts "=> Unable to load #{url}; #{error.class}: #{error.message}"
   []
 end
 
-domains = []
 threads = []
 
-threads << thread { domains += load_github_url("https://raw.githubusercontent.com/ivolo/disposable-email-domains/master/index.json") }
-threads << thread { domains += load_github_url("https://raw.githubusercontent.com/andreis/disposable-email-domains/master/domains.json") }
-threads << thread { domains += load_github_url("https://raw.githubusercontent.com/FGRibreau/mailchecker/master/list.txt") }
-threads << thread { domains += load_github_url("https://raw.githubusercontent.com/willwhite/freemail/master/data/disposable.txt") }
-threads << thread { domains += load_github_url("https://raw.githubusercontent.com/maxmalysh/disposable-emails/master/disposable_emails/data/domains.txt") }
-threads << thread { domains += load_github_url("https://raw.githubusercontent.com/sneakykiwi/LeagueCreatorPublic/master/emails.txt") }
-threads << thread { domains += load_github_url("https://raw.githubusercontent.com/jespernissen/disposable-maildomain-list/master/disposable-maildomain-list.txt") }
-threads << thread { domains += load_github_url("https://raw.githubusercontent.com/wesbos/burner-email-providers/master/emails.txt") }
-threads << thread { domains += load_github_url("https://gist.github.com/fnando/dafe542cac13f831bbf5521a55248116/raw/f364d880cee5c878531e4f48be1744ff6ec84cb8/disposable.txt") }
-threads << thread { domains += ten_minute_mail }
-threads << thread { domains += temp_mail }
-threads << thread { domains += temp_mail_address }
-threads << thread { domains += tempmail_io }
-threads << thread { domains += load_file("disposable/disposable_manually_added.json") }
-threads << thread { domains += domain_scraping("guerrillamail", "https://www.guerrillamail.com/", "select option::attr(value)") }
-threads << thread { domains += domain_scraping("moakt", "https://www.moakt.com", "select option::attr(value)") }
-threads << thread { domains += domain_scraping("tempr", "https://tempr.email/", "select[name=DomainId] option::text()") }
-threads << thread { domains += domain_scraping("ically", "https://ically.net/", "select[name=domain] option::text()") }
-threads << thread { domains += domain_scraping("yepmail", "https://yepmail.co/", "select[name=domain] option::text()") }
-threads << thread { domains += domain_scraping("fake_email_generator", "https://fakemailgenerator.net", "[data-mailhost]::attr(data-mailhost)") }
-threads << thread { domains += domain_scraping("tempemails", "https://www.tempemails.net/", "select[name=domain] option::attr(value)") }
-threads << thread { domains += domain_scraping("clipmails", "https://clipmails.com/", "select[name=domain] option::attr(value)") }
-threads << thread { domains += domain_scraping("1secmail", "https://www.1secmail.com/", "select[id=domain] option::attr(value)") }
-threads << thread { domains += domain_scraping("emailfake", "https://generator.email", ".tt-suggestion p::text()") }
-threads << thread { domains += domain_scraping("emailfake", "https://emailfake.com/", ".tt-suggestion p::text()") }
-threads << thread { domains += domain_scraping("emailfake", "https://email-fake.com/", ".tt-suggestion p::text()") }
-threads << thread { domains += domain_scraping("receivemail", "https://www.receivemail.org/", "select[name=domain] option::text()") }
-threads << thread { domains += domain_scraping("itemp", "https://itemp.email", "select[name=domain] option::text()") }
-threads << thread { domains += domain_scraping("tempomail", "https://tempomail.org", "select[name=domain] option::text()") }
-threads << thread { domains += domain_scraping("tempmail", "https://tempmail.top", "select[name=domain] option::text()") }
-threads << thread { domains += domain_scraping("cs", "https://www.cs.email", "select[id=gm-host-select] option::text()") }
-threads << thread { domains += domain_scraping("tempmail", "https://tempmail.io/settings/", "select[id=domain] option::text()") }
-threads << thread { domains += domain_scraping("tempemail", "https://tempemail.co", "select[name=email_domain] option::text()") }
-threads << thread { domains += domain_scraping("tmail", "https://mytemp-email.com/", "a.domain-selector::text()") }
+threads << thread { load_github_url("https://raw.githubusercontent.com/ivolo/disposable-email-domains/master/index.json") }
+threads << thread { load_github_url("https://raw.githubusercontent.com/andreis/disposable-email-domains/master/domains.json") }
+threads << thread { load_github_url("https://raw.githubusercontent.com/FGRibreau/mailchecker/master/list.txt") }
+threads << thread { load_github_url("https://raw.githubusercontent.com/willwhite/freemail/master/data/disposable.txt") }
+threads << thread { load_github_url("https://raw.githubusercontent.com/maxmalysh/disposable-emails/master/disposable_emails/data/domains.txt") }
+threads << thread { load_github_url("https://raw.githubusercontent.com/jespernissen/disposable-maildomain-list/master/disposable-maildomain-list.txt") }
+threads << thread { load_github_url("https://raw.githubusercontent.com/wesbos/burner-email-providers/master/emails.txt") }
+threads << thread { load_github_url("https://gist.github.com/fnando/dafe542cac13f831bbf5521a55248116/raw/disposable.txt") }
+threads << thread { ten_minute_mail }
+threads << thread { temp_mail }
+threads << thread { temp_mail_address }
+threads << thread { tempmail_io }
+threads << thread { load_file("disposable/disposable_manually_added.txt") }
+threads << thread { domain_scraping("guerrillamail", "https://www.guerrillamail.com/", "select option::attr(value)") }
+threads << thread { domain_scraping("moakt", "https://www.moakt.com", "select option::attr(value)") }
+threads << thread { domain_scraping("tempr", "https://tempr.email/", "select[name=DomainId] option::text()") }
+threads << thread { domain_scraping("yepmail", "https://yepmail.co/", "select[name=domain] option::text()") }
+threads << thread { domain_scraping("fake_email_generator", "https://fakemailgenerator.net", "[data-mailhost]::attr(data-mailhost)") }
+threads << thread { domain_scraping("tempemails", "https://www.tempemails.net/", "select[name=domain] option::attr(value)") }
+threads << thread { domain_scraping("clipmails", "https://clipmails.com/", "select[name=domain] option::attr(value)") }
+threads << thread { domain_scraping("1secmail", "https://www.1secmail.com/", "select[id=domain] option::attr(value)") }
+threads << thread { domain_scraping("emailfake", "https://generator.email", ".tt-suggestion p::text()") }
+threads << thread { domain_scraping("emailfake", "https://emailfake.com/", ".tt-suggestion p::text()") }
+threads << thread { domain_scraping("emailfake", "https://email-fake.com/", ".tt-suggestion p::text()") }
+threads << thread { domain_scraping("receivemail", "https://www.receivemail.org/", "select[name=domain] option::text()") }
+threads << thread { domain_scraping("itemp", "https://itemp.email", "select[name=domain] option::text()") }
+threads << thread { domain_scraping("cs", "https://www.cs.email", "select[id=gm-host-select] option::text()") }
+threads << thread { domain_scraping("tempmail", "https://tempmail.io/settings/", "select[id=domain] option::text()") }
+threads << thread { domain_scraping("tempemail", "https://tempemail.co", "select[name=email_domain] option::text()") }
+threads << thread { domain_scraping("tmail", "https://mytemp-email.com/", "a.domain-selector::text()") }
 
 threads.each_slice(5) do |slice|
   slice.each(&:join)
@@ -204,17 +201,30 @@ end
 
 threads.clear
 
-# Unprocessed domains
-save_file("disposable_raw.json", domains)
+domains = []
+
+puts "=> Loading disposable_domains.txt"
+domains += File.read("#{__dir__}/../data/disposable_domains.txt").lines.map(&:chomp)
+
+puts "=> Loading disposable/*.txt"
+Dir["./data/disposable/**/*.txt"].map do |file|
+  file = File.expand_path(file)
+  domains += File.read(file).lines.map(&:chomp).flatten.compact
+end
 
-domains.map!(&:downcase)
-domains = root_domains(domains)
+ignore_domains = %w[gmail.com hotmail.com]
 
-# All disposable domains, unfiltered
-save_file("disposable.json", domains)
+puts "=> Normalize domains (count: #{domains.size})"
+domains = domains
+          .uniq
+          .map {|domain| RootDomain.call(domain.split("@").last.downcase) }
+          .compact
+          .uniq
+          .reject {|domain| ignore_domains.include?(domain) }
 
-# All disposable domains, with patterns
-save_file("disposable_patterns.json", prepare_patterns(domains))
+puts "=> Saving domains (count: #{domains.size})"
+save_file("disposable_domains.txt", domains)
 
-# Emails being used as proxy.
-save_file("disposable_emails.json", gmailnator)
+emails = gmailnator
+puts "=> Saving email proxies (count: #{emails.size})"
+save_file("disposable_emails.txt", emails)

data/bin/sync-tld

@@ -10,11 +10,11 @@ tlds.map!(&:downcase)
 tlds.map!(&:strip)
 tlds.map! {|tld| SimpleIDN.to_ascii(tld) }
 
-save_file("tld.json", tlds)
+save_file("tld.txt", tlds)
 
 country_tlds = JSON.parse(http_request(:get, "https://github.com/samayo/country-json/raw/master/src/country-by-domain-tld.json").body, symbolize_names: true)
 country_tlds = country_tlds
                .reject {|info| info[:tld].nil? }
                .map {|info| info[:tld].gsub(/^\./, "") }
 
-save_file("country_tlds.json", country_tlds)
+save_file("country_tlds.txt", country_tlds)

data/data/country_tlds.txt

@@ -0,0 +1,235 @@
+ad
+ae
+af
+ag
+ai
+al
+am
+an
+ao
+aq
+ar
+as
+at
+au
+aw
+az
+ba
+bb
+bd
+be
+bf
+bg
+bh
+bi
+bj
+bm
+bn
+bo
+br
+bs
+bt
+bv
+bw
+by
+bz
+ca
+cc
+cd
+cf
+cg
+ch
+ci
+ck
+cl
+cm
+cn
+co
+cr
+cu
+cv
+cx
+cy
+cz
+de
+dj
+dk
+dm
+do
+dz
+ec
+ee
+eg
+eh
+er
+es
+et
+fi
+fj
+fk
+fr
+ga
+gb
+gd
+ge
+gf
+gh
+gi
+gl
+gm
+gn
+gp
+gq
+gr
+gs
+gt
+gu
+gw
+gy
+hk
+hm
+hn
+hr
+ht
+hu
+id
+ie
+il
+in
+io
+iq
+ir
+is
+it
+jm
+jo
+jp
+ke
+kg
+kh
+ki
+km
+kn
+kp
+kr
+kw
+ky
+kz
+la
+lb
+lc
+li
+lk
+lr
+ls
+lt
+lu
+lv
+ly
+ma
+mc
+md
+mg
+mh
+mk
+ml
+mm
+mn
+mo
+mp
+mq
+mr
+ms
+mt
+mu
+mv
+mw
+mx
+my
+mz
+na
+nc
+ne
+nf
+ng
+ni
+nl
+no
+np
+nr
+nu
+nz
+om
+pa
+pe
+pf
+pg
+ph
+pk
+pl
+pm
+pn
+pr
+ps
+pt
+pw
+py
+qa
+re
+ro
+ru
+rw
+sa
+sb
+sc
+sd
+se
+sg
+sh
+si
+sj
+sk
+sl
+sm
+sn
+so
+sr
+ss
+st
+sv
+sy
+sz
+tc
+td
+tf
+tg
+th
+tj
+tk
+tl
+tm
+tn
+to
+tr
+tt
+tv
+tz
+ua
+ug
+us
+uy
+uz
+va
+vc
+ve
+vg
+vi
+vn
+vu
+wf
+ws
+ye
+yt
+za
+zm
+zw