validators 3.1.1 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 25a774489dab7d08065e4fd32aae5540c7539ad2089096074e0e9bddfdb6d409
-  data.tar.gz: bf68b6e29ab10f72a01db549003ae545f09ca03b9b6e727d85bd99f773edacd2
+  metadata.gz: a9866f3a757a7bc59c1ea77da0ad9bd8f6c375f7caad128c0abcecfb596ed56a
+  data.tar.gz: d4e52e82630413c929e5c007ae31dd7667c128ec8201850546d8ba331148b5ed
 SHA512:
-  metadata.gz: b01421cbb3d76ddfc08327c4c86ab9d1af981f4f32c7648146c715978029345c09ef7428a6e9917e4eae2605ca12121f9aa5930a83a5ddaaf5eaeb8204c62455
-  data.tar.gz: e1f6d616018639bec217b77c3ceb94a0fa444eac20b46833f00c5f809d53c5c1c0d8a45bbf690c14ad7790cc460364e901e4434010b524f65bd29fc55844a5f1
+  metadata.gz: 11b91772a301abd780957d7697f75bc41da64a43ef6a55d2a38a298522571f9117b9cec1ac1403b0cf9e56e825e3cd1dc7dac3471ca77ab6823790f2c0d96c84
+  data.tar.gz: 2cafe7d2fc93d7ca8e26d0317857b959ca9070f6240f5b74e27e9c8b4706b9f727d7a18ebf3a610579d20ee7d617637b1896686609f23e332c2914c0a4378320

data/.rubocop.yml

@@ -28,3 +28,8 @@ Style/IfUnlessModifier:
 
 Metrics/MethodLength:
   Enabled: false
+
+Metrics/BlockLength:
+  Exclude:
+    - bin/**/*
+    - "*.gemspec"

data/README.md

@@ -144,17 +144,24 @@ class Server < ActiveRecord::Base
 end
 ```
 
-### validates_reserved_username / validates_reserved_hostname
+### validates_username / validates_subdomain
 
-The compiled list will be used for both username and hostname validations.
+A valid username/subdomain follows the hostname label validation:
+
+- maximum length is 63 characters
+- allowed characters are a-z, A-Z, 0-9 and hyphen
+- cannot begin or end with a hyphen
+- cannot consist of numeric values only
+
+The compiled list will be used for both username and subdomain validations.
 
 ```ruby
 class Server < ActiveRecord::Base
-  validates_reserved_hostname :hostname
+  validates_subdomain :subdomain
 end
 
 class User < ActiveRecord::Base
-  validates_reserved_username :username
+  validates_username :username
 end
 ```
 
@@ -168,7 +175,15 @@ ReservedUsernames = Validators::ReservedHostnames.parse_list([
 ])
 
 class User < ActiveRecord::Base
-  validates_reserved_username, in: ReservedUsernames
+  validates_username :username, in: ReservedUsernames
+end
+```
+
+To disable the reserved validation, use `reserved: false`.
+
+```ruby
+class User < ActiveRecord::Base
+  validates_username :username, reserved: false
 end
 ```
 

data/bin/sync-disposable-hostnames

@@ -1,35 +1,220 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-require "open-uri"
-require "json"
+require_relative "helpers"
 
-urls = %w[
-  https://raw.githubusercontent.com/ivolo/disposable-email-domains/master/index.json
-  https://raw.githubusercontent.com/andreis/disposable-email-domains/master/domains.json
-  https://raw.githubusercontent.com/FGRibreau/mailchecker/master/list.txt
-  https://raw.githubusercontent.com/willwhite/freemail/master/data/disposable.txt
-]
+def ten_minute_mail
+  path = "disposable/10minutemail.json"
+  url = "https://10minutemail.com/10MinuteMail/resources/session/address"
+
+  refresh_list(url: url, path: path) do |response|
+    _account, host = response.body.split("@")
+
+    [host]
+  end
+end
+
+def temp_mail
+  path = "disposable/tempmail.json"
+  url = "https://api4.temp-mail.org/request/domains/format/json"
+
+  refresh_list(url: url, path: path) do |response|
+    response.data.map {|domain| domain.tr("@", "") }
+  end
+end
+
+def temp_mail_address
+  path = "disposable/tempmailaddress.json"
+  url = "https://www.tempmailaddress.com/index/index"
+
+  refresh_list(url: url, path: path) do |response|
+    data = JSON.parse(
+      response.body.gsub(/[^-,:\w@.{}"]/, ""),
+      symbolize_names: true
+    )
+    [data[:email].split("@").last]
+  end
+end
+
+def tempmail_io
+  path = "disposable/tempmail_io.json"
+  url = "https://api.internal.temp-mail.io/api/v2/domains"
+
+  refresh_list(url: url, path: path) do |response|
+    response.data["domains"]
+  end
+end
+
+def gmailnator
+  path = "disposable/gmailnator.json"
+  url = "https://gmailnator.com/index/indexquery"
+
+  refresh_list(
+    verb: :post,
+    url: url,
+    path: path,
+    params: {action: "GenerateEmail"}
+  ) do |response|
+    email = response.body.gsub(/(\+[^@]+)/, "")
+    [email]
+  end
+end
+
+def prepare_patterns(domains)
+  new_domains = domains.map do |domain|
+    {
+      domain: domain,
+      processed: domain.gsub(/\..*?$/, "")
+    }
+  end
+
+  stats = new_domains
+          .count_by {|info| info[:processed] }
+          .select {|_processed, count| count > 2 }
+
+  stats_map = stats.each_with_object({}) do |(name, count), buffer|
+    buffer[name] = count
+  end
+
+  domains = new_domains
+            .reject {|info| stats_map[info[:processed]] }
+            .map {|info| info[:domain] }
+  domains += stats_map.keys.map {|processed| "/#{processed}\\..+$/" }
+
+  domains.select {|domain| domain.start_with?("/") }
+end
+
+def domain_scraping(name, url, selector)
+  puts "=> scraping #{url}"
+
+  selector, value_selector = selector.split("::")
+  path = "disposable/#{name}.json"
+  host_regex = /@?(.*?(\.[^.]+)+)/
+
+  refresh_list(url: url, path: path) do |response|
+    new_domains = response
+                  .data
+                  .css(selector)
+                  .map {|element| process_scraping(element, value_selector) }
+
+    new_domains = new_domains
+                  .map(&:squeeze)
+                  .map(&:strip)
+                  .reject(&:empty?)
+                  .map {|domain| domain[host_regex, 1]&.squeeze&.tr("@", "") }
+                  .reject(&:nil?)
+                  .reject(&:empty?)
+                  .map {|domain| domain.gsub(/\s*\((.*?)\)/, "") }
+
+    raise "No #{name} hosts found" if new_domains.empty?
+
+    new_domains
+  end
+rescue StandardError => error
+  puts "=> [ERROR] Unable to scrape #{url}; #{error.class}: #{error.message}"
+  []
+end
+
+def process_scraping(element, value_selector)
+  value = nil
+
+  case value_selector
+  when "text()"
+    value = element.text
+  when /^attr\((.*?)\)/
+    value = element[Regexp.last_match(1)]
+  else
+    element.attributes.each do |_name, attr|
+      attr = attr.value.to_s
+      value = attr if attr =~ host_regex
+    end
+  end
+
+  raise "no value found: #{element} (value_selector: #{value_selector})" unless value
+
+  value
+end
+
+def load_github_url(url)
+  puts "=> Fetching #{url}"
+
+  basename = URI.parse(url).path[%r{/([^/]+/[^/]+)}, 1].tr("/", "_").tr("-", "_")
+  path = "disposable/#{basename}.json"
+  domains = load_file(path)
 
-domains = urls.each_with_object([]) do |url, buffer|
   ext = File.extname(url)
 
-  result = case ext
-           when ".json"
-             JSON.parse(URI.open(url).read)
-           when ".txt"
-             URI.open(url).read.lines.map(&:chomp)
-           else
-             raise "Unknown extension"
-           end
+  domains += case ext
+             when ".json"
+               JSON.parse(http_request(:get, url).body)
+             when ".txt"
+               http_request(:get, url).body.lines.map(&:chomp)
+             else
+               raise "Unknown extension"
+             end
 
-  buffer.push(*result)
+  append_to_file(path, domains)
+  domains
+rescue error
+  puts "=> Unable to load #{url}; #{error.class}: #{error.message}"
+  []
 end
 
-domains.map!(&:downcase)
-domains.uniq!
-domains.sort!
+domains = []
+threads = []
 
-File.open("./data/disposable.json", "w") do |file|
-  file << JSON.pretty_generate(domains)
+threads << thread { domains += load_github_url("https://raw.githubusercontent.com/ivolo/disposable-email-domains/master/index.json") }
+threads << thread { domains += load_github_url("https://raw.githubusercontent.com/andreis/disposable-email-domains/master/domains.json") }
+threads << thread { domains += load_github_url("https://raw.githubusercontent.com/FGRibreau/mailchecker/master/list.txt") }
+threads << thread { domains += load_github_url("https://raw.githubusercontent.com/willwhite/freemail/master/data/disposable.txt") }
+threads << thread { domains += load_github_url("https://raw.githubusercontent.com/maxmalysh/disposable-emails/master/disposable_emails/data/domains.txt") }
+threads << thread { domains += load_github_url("https://raw.githubusercontent.com/sneakykiwi/LeagueCreatorPublic/master/emails.txt") }
+threads << thread { domains += load_github_url("https://raw.githubusercontent.com/jespernissen/disposable-maildomain-list/master/disposable-maildomain-list.txt") }
+threads << thread { domains += load_github_url("https://raw.githubusercontent.com/wesbos/burner-email-providers/master/emails.txt") }
+threads << thread { domains += load_github_url("https://gist.github.com/fnando/dafe542cac13f831bbf5521a55248116/raw/f364d880cee5c878531e4f48be1744ff6ec84cb8/disposable.txt") }
+threads << thread { domains += ten_minute_mail }
+threads << thread { domains += temp_mail }
+threads << thread { domains += temp_mail_address }
+threads << thread { domains += tempmail_io }
+threads << thread { domains += load_file("disposable/disposable_manually_added.json") }
+threads << thread { domains += domain_scraping("guerrillamail", "https://www.guerrillamail.com/", "select option::attr(value)") }
+threads << thread { domains += domain_scraping("moakt", "https://www.moakt.com", "select option::attr(value)") }
+threads << thread { domains += domain_scraping("tempr", "https://tempr.email/", "select[name=DomainId] option::text()") }
+threads << thread { domains += domain_scraping("ically", "https://ically.net/", "select[name=domain] option::text()") }
+threads << thread { domains += domain_scraping("yepmail", "https://yepmail.co/", "select[name=domain] option::text()") }
+threads << thread { domains += domain_scraping("fake_email_generator", "https://fakemailgenerator.net", "[data-mailhost]::attr(data-mailhost)") }
+threads << thread { domains += domain_scraping("tempemails", "https://www.tempemails.net/", "select[name=domain] option::attr(value)") }
+threads << thread { domains += domain_scraping("clipmails", "https://clipmails.com/", "select[name=domain] option::attr(value)") }
+threads << thread { domains += domain_scraping("1secmail", "https://www.1secmail.com/", "select[id=domain] option::attr(value)") }
+threads << thread { domains += domain_scraping("emailfake", "https://generator.email", ".tt-suggestion p::text()") }
+threads << thread { domains += domain_scraping("emailfake", "https://emailfake.com/", ".tt-suggestion p::text()") }
+threads << thread { domains += domain_scraping("emailfake", "https://email-fake.com/", ".tt-suggestion p::text()") }
+threads << thread { domains += domain_scraping("receivemail", "https://www.receivemail.org/", "select[name=domain] option::text()") }
+threads << thread { domains += domain_scraping("itemp", "https://itemp.email", "select[name=domain] option::text()") }
+threads << thread { domains += domain_scraping("tempomail", "https://tempomail.org", "select[name=domain] option::text()") }
+threads << thread { domains += domain_scraping("tempmail", "https://tempmail.top", "select[name=domain] option::text()") }
+threads << thread { domains += domain_scraping("cs", "https://www.cs.email", "select[id=gm-host-select] option::text()") }
+threads << thread { domains += domain_scraping("tempmail", "https://tempmail.io/settings/", "select[id=domain] option::text()") }
+threads << thread { domains += domain_scraping("tempemail", "https://tempemail.co", "select[name=email_domain] option::text()") }
+threads << thread { domains += domain_scraping("tmail", "https://mytemp-email.com/", "a.domain-selector::text()") }
+
+threads.each_slice(5) do |slice|
+  slice.each(&:join)
 end
+
+threads.clear
+
+# Unprocessed domains
+save_file("disposable_raw.json", domains)
+
+domains.map!(&:downcase)
+domains = root_domains(domains)
+
+# All disposable domains, unfiltered
+save_file("disposable.json", domains)
+
+# All disposable domains, with patterns
+save_file("disposable_patterns.json", prepare_patterns(domains))
+
+# Emails being used as proxy.
+save_file("disposable_emails.json", gmailnator)

data/bin/sync-tld

@@ -1,17 +1,20 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-require "open-uri"
-require "json"
+require_relative "helpers"
 
-tlds = URI.open("https://data.iana.org/TLD/tlds-alpha-by-domain.txt").read.lines
+tlds = http_request(:get, "https://data.iana.org/TLD/tlds-alpha-by-domain.txt").body.lines
 tlds.shift # remove update notice
 
 tlds.map!(&:downcase)
 tlds.map!(&:strip)
-tlds.sort!
-tlds.uniq!
+tlds.map! {|tld| SimpleIDN.to_ascii(tld) }
 
-File.open("./data/tld.json", "w") do |file|
-  file << JSON.pretty_generate(tlds)
-end
+save_file("tld.json", tlds)
+
+country_tlds = JSON.parse(http_request(:get, "https://github.com/samayo/country-json/raw/master/src/country-by-domain-tld.json").body, symbolize_names: true)
+country_tlds = country_tlds
+               .reject {|info| info[:tld].nil? }
+               .map {|info| info[:tld].gsub(/^\./, "") }
+
+save_file("country_tlds.json", country_tlds)

data/data/country_tlds.json

@@ -0,0 +1,237 @@
+[
+  "ad",
+  "ae",
+  "af",
+  "ag",
+  "ai",
+  "al",
+  "am",
+  "an",
+  "ao",
+  "aq",
+  "ar",
+  "as",
+  "at",
+  "au",
+  "aw",
+  "az",
+  "ba",
+  "bb",
+  "bd",
+  "be",
+  "bf",
+  "bg",
+  "bh",
+  "bi",
+  "bj",
+  "bm",
+  "bn",
+  "bo",
+  "br",
+  "bs",
+  "bt",
+  "bv",
+  "bw",
+  "by",
+  "bz",
+  "ca",
+  "cc",
+  "cd",
+  "cf",
+  "cg",
+  "ch",
+  "ci",
+  "ck",
+  "cl",
+  "cm",
+  "cn",
+  "co",
+  "cr",
+  "cu",
+  "cv",
+  "cx",
+  "cy",
+  "cz",
+  "de",
+  "dj",
+  "dk",
+  "dm",
+  "do",
+  "dz",
+  "ec",
+  "ee",
+  "eg",
+  "eh",
+  "er",
+  "es",
+  "et",
+  "fi",
+  "fj",
+  "fk",
+  "fr",
+  "ga",
+  "gb",
+  "gd",
+  "ge",
+  "gf",
+  "gh",
+  "gi",
+  "gl",
+  "gm",
+  "gn",
+  "gp",
+  "gq",
+  "gr",
+  "gs",
+  "gt",
+  "gu",
+  "gw",
+  "gy",
+  "hk",
+  "hm",
+  "hn",
+  "hr",
+  "ht",
+  "hu",
+  "id",
+  "ie",
+  "il",
+  "in",
+  "io",
+  "iq",
+  "ir",
+  "is",
+  "it",
+  "jm",
+  "jo",
+  "jp",
+  "ke",
+  "kg",
+  "kh",
+  "ki",
+  "km",
+  "kn",
+  "kp",
+  "kr",
+  "kw",
+  "ky",
+  "kz",
+  "la",
+  "lb",
+  "lc",
+  "li",
+  "lk",
+  "lr",
+  "ls",
+  "lt",
+  "lu",
+  "lv",
+  "ly",
+  "ma",
+  "mc",
+  "md",
+  "mg",
+  "mh",
+  "mk",
+  "ml",
+  "mm",
+  "mn",
+  "mo",
+  "mp",
+  "mq",
+  "mr",
+  "ms",
+  "mt",
+  "mu",
+  "mv",
+  "mw",
+  "mx",
+  "my",
+  "mz",
+  "na",
+  "nc",
+  "ne",
+  "nf",
+  "ng",
+  "ni",
+  "nl",
+  "no",
+  "np",
+  "nr",
+  "nu",
+  "nz",
+  "om",
+  "pa",
+  "pe",
+  "pf",
+  "pg",
+  "ph",
+  "pk",
+  "pl",
+  "pm",
+  "pn",
+  "pr",
+  "ps",
+  "pt",
+  "pw",
+  "py",
+  "qa",
+  "re",
+  "ro",
+  "ru",
+  "rw",
+  "sa",
+  "sb",
+  "sc",
+  "sd",
+  "se",
+  "sg",
+  "sh",
+  "si",
+  "sj",
+  "sk",
+  "sl",
+  "sm",
+  "sn",
+  "so",
+  "sr",
+  "ss",
+  "st",
+  "sv",
+  "sy",
+  "sz",
+  "tc",
+  "td",
+  "tf",
+  "tg",
+  "th",
+  "tj",
+  "tk",
+  "tl",
+  "tm",
+  "tn",
+  "to",
+  "tr",
+  "tt",
+  "tv",
+  "tz",
+  "ua",
+  "ug",
+  "us",
+  "uy",
+  "uz",
+  "va",
+  "vc",
+  "ve",
+  "vg",
+  "vi",
+  "vn",
+  "vu",
+  "wf",
+  "ws",
+  "ye",
+  "yt",
+  "za",
+  "zm",
+  "zw"
+]