email_assessor 0.9.1 → 1.0.0

data/assets/data/educational_domains/v.txt

@@ -0,0 +1,78 @@
+va.lv
+valahia.ro
+valleystudent.org
+vaniercollege.qc.ca
+vansd.org
+vbu.co.in
+vbu.se
+vcc.ca
+vdu.lt
+veda-edu.com
+venturaedu.org
+ver.ucc.mx
+vern.hr
+vet-alfort.fr
+vet-lyon.fr
+vet-nantes.fr
+veths.no
+vfrta.ru
+vfu.bg
+vfu.cz
+vgafk.ru
+vgasa.ru
+vgmu.vitebsk.by
+vgta.vrn.ru
+vgtu.lt
+victoria-uni.ch
+victoryedu.com
+vicu.utoronto.ca
+vieup.ru
+vignanuniversity.org
+vikramuniversity.org
+vinayakamissions.com
+vineland.org
+vkgu.kz
+vksu-ara.org
+vlekho.be
+vlerick.be
+vlsu.ru
+vmchs.com
+vnmu.vn.ua
+voenmeh.ru
+volsu.ru
+vorskoler.com
+vorskoler.net
+vpu.lt
+vsau.ru
+vsavm.com
+vsb.cz
+vscht.cz
+vsci.cz
+vse.cz
+vsgaki.burnet.ru
+vsgtu.eastsib.ru
+vslib.cz
+vsm.sk
+vsma.info
+vsmu.sk
+vsp.cz
+vspu.kirov.ru
+vspu.ru
+vssladkovicovo.sk
+vssvalzbety.sk
+vstu.ru
+vstu.vinnica.ua
+vstu.vitebsk.by
+vsu.by
+vsu.ru
+vsvu.sk
+vu.lt
+vu.nl
+vub.be
+vumk.eu
+vusd.us
+vut.cz
+vutbr.cz
+vvsd.org
+vvsu.ru
+vxu.se

data/assets/data/educational_domains/w.txt

@@ -0,0 +1,59 @@
+wabash348.com
+wageningenuniversity.nl
+waitakerecollege.school.nz
+wakf.org
+warrenk12nc.org
+warrensd.org
+washk12.org
+watfordboys.org
+wbuafs.nic.in
+wbut.net
+wcs-g.com
+wcusd.net
+wdsdjxy.com
+web.prover.com.br
+webs.satlink.com
+webster.ch
+wenk.be
+westburyschools.org
+westcoastuniversity-edu.com
+westcoastuniversity.bz
+westfieldacademy.co.uk
+westhillscollege.com
+westmorelandschool.org
+westorangeschools.org
+whu-koblenz.de
+williamgilbert.co.uk
+willistonschools.org
+wilsonk12tn.us
+windesheim.nl
+wit.ie
+wittenborg.eu
+wiut.uz
+wkau.kz
+wlodkowic.pl
+wlu.ca
+wmsd.net
+wmu.se
+worcesterschools.net
+wpsstudent.com
+wsb.poznan.pl
+wsb.toi.tarnow.pl
+wsdoc.me
+wsei.lublin.pl
+wsiz.rzeszow.pl
+wsm.gdynia.pl
+wsm.szczecin.pl
+wsp.bydgoszcz.pl
+wsp.czest.pl
+wsp.krakow.pl
+wsp.slupsk.pl
+wsp.zgora.pl
+wsps.waw.pl
+wsub.waw.pl
+wsz.pl
+wszib.krakow.pl
+wvu.edu
+wwu.de
+www2.mozcom.com
+wzmu.net

data/assets/data/educational_domains/x.txt

@@ -0,0 +1 @@
+xza.cn

data/assets/data/educational_domains/y.txt

@@ -0,0 +1,17 @@
+yandex.ru
+yarmouthschools.org
+ycdsbk12.ca
+ycmou.com
+yizhuan.com
+yorku.ca
+yppsb.id
+yrdsb.ca
+ysmu.am
+ysparmaruniversity.org
+yspu.yar.ru
+ystu.yar.ru
+ysu.am
+ysu.ru
+yukoncollege.yk.ca
+yuniv.net
+yznu.cn

data/assets/data/educational_domains/z.txt

@@ -0,0 +1,13 @@
+zcu.cz
+zdsoft.com.cn
+zetechcollege.com
+zgfxy.cn
+zhaw.ch
+zhdk.ch
+zhezu.kz
+zhgu.kz
+zhwin.ch
+znuel.net
+zpsb.szczecin.pl
+zsem.hr
+zuyd.nl

data/assets/data/fastpass_domains/_prioritization.txt

@@ -0,0 +1 @@
+dfirtawcpsmglohy

data/assets/data/fastpass_domains/a.txt

@@ -0,0 +1,2 @@
+aol.com
+att.net

data/assets/data/fastpass_domains/c.txt

@@ -0,0 +1,3 @@
+citromail.hu
+comcast.net
+cox.net

data/assets/data/fastpass_domains/d.txt

@@ -0,0 +1 @@
+duck.com

data/assets/data/fastpass_domains/f.txt

@@ -0,0 +1 @@
+fastmail.com

data/assets/data/fastpass_domains/g.txt

@@ -0,0 +1,5 @@
+gmail.com
+gmx.at
+gmx.de
+gmx.fr
+googlemail.com

data/assets/data/fastpass_domains/h.txt

@@ -0,0 +1,33 @@
+hotmail.be
+hotmail.ca
+hotmail.ch
+hotmail.cl
+hotmail.co.id
+hotmail.co.jp
+hotmail.co.kr
+hotmail.co.nz
+hotmail.co.th
+hotmail.co.uk
+hotmail.co.za
+hotmail.com
+hotmail.com.ar
+hotmail.com.au
+hotmail.com.br
+hotmail.com.tr
+hotmail.com.tw
+hotmail.de
+hotmail.dk
+hotmail.es
+hotmail.fi
+hotmail.fr
+hotmail.gr
+hotmail.hu
+hotmail.it
+hotmail.my
+hotmail.nl
+hotmail.no
+hotmail.ph
+hotmail.rs
+hotmail.se
+hotmail.sg
+hotmail.sk

data/assets/data/fastpass_domains/i.txt

@@ -0,0 +1 @@
+icloud.com

data/assets/data/fastpass_domains/l.txt

@@ -0,0 +1,6 @@
+live.co.uk
+live.com
+live.com.mx
+live.de
+live.fr
+live.nl

data/assets/data/fastpass_domains/m.txt

@@ -0,0 +1,4 @@
+mail.com
+mail.ru
+me.com
+msn.com

data/assets/data/fastpass_domains/o.txt

@@ -0,0 +1,28 @@
+o2.pl
+orange.fr
+outlook.at
+outlook.be
+outlook.co.id
+outlook.co.nz
+outlook.co.th
+outlook.com
+outlook.com.ar
+outlook.com.au
+outlook.com.br
+outlook.com.tr
+outlook.com.vn
+outlook.cz
+outlook.de
+outlook.dk
+outlook.es
+outlook.fr
+outlook.hu
+outlook.in
+outlook.it
+outlook.jp
+outlook.kr
+outlook.ph
+outlook.pt
+outlook.sa
+outlook.sg
+outlook.sk

data/assets/data/fastpass_domains/p.txt

@@ -0,0 +1,3 @@
+pm.me
+proton.me
+protonmail.com

data/assets/data/fastpass_domains/r.txt

@@ -0,0 +1 @@
+rocketmail.com

data/assets/data/fastpass_domains/s.txt

@@ -0,0 +1,3 @@
+sbcglobal.net
+seznam.cz
+simplelogin.com

data/assets/data/fastpass_domains/t.txt

@@ -0,0 +1 @@
+tuta.io

data/assets/data/fastpass_domains/w.txt

@@ -0,0 +1,2 @@
+web.de
+wp.pl

data/assets/data/fastpass_domains/y.txt

@@ -0,0 +1,42 @@
+yahoo.at
+yahoo.be
+yahoo.ca
+yahoo.cl
+yahoo.co.id
+yahoo.co.in
+yahoo.co.jp
+yahoo.co.nz
+yahoo.co.th
+yahoo.co.uk
+yahoo.co.za
+yahoo.com
+yahoo.com.ar
+yahoo.com.au
+yahoo.com.br
+yahoo.com.co
+yahoo.com.hk
+yahoo.com.hr
+yahoo.com.mx
+yahoo.com.my
+yahoo.com.ph
+yahoo.com.sg
+yahoo.com.tw
+yahoo.com.ve
+yahoo.com.vn
+yahoo.de
+yahoo.dk
+yahoo.es
+yahoo.fr
+yahoo.gr
+yahoo.hu
+yahoo.ie
+yahoo.in
+yahoo.it
+yahoo.ne.jp
+yahoo.nl
+yahoo.no
+yahoo.pl
+yahoo.ro
+yahoo.se
+yandex.com
+ymail.com

data/lib/email_assessor/address.rb

@@ -115,38 +115,25 @@ module EmailAssessor
     end
 
     def disposable?
-      domain_in_file?(EmailAssessor::DISPOSABLE_DOMAINS_FILE_NAME)
+      domain_in_list?(EmailAssessor.disposable_domains)
     end
 
     def blacklisted?
-      domain_in_file?(EmailAssessor::BLACKLISTED_DOMAINS_FILE_NAME)
+      domain_in_list?(EmailAssessor.blacklisted_domains)
     end
 
     def educational?
-      domain_in_file?(EmailAssessor::EDUCATIONAL_DOMAINS_FILE_NAME)
+      domain_in_list?(EmailAssessor.educational_domains)
     end
 
     def fastpass?
-      domain_in_file?(EmailAssessor::FASTPASS_DOMAINS_FILE_NAME)
-    end
-
-    def domain_in_file?(filename)
-      valid? && EmailAssessor.any_token_in_file?(domain_tokens, filename)
+      domain_in_list?(EmailAssessor.fastpass_domains)
     end
 
     def valid_mx?
       valid? && mx_servers.any?
     end
 
-    def mx_server_is_in?(domain_list_file)
-      mx_servers.any? do |mx_server|
-        return false unless mx_server.respond_to?(:exchange)
-        mx_server = mx_server.exchange.to_s
-
-        EmailAssessor.domain_in_file?(mx_server, domain_list_file)
-      end
-    end
-
     def mx_servers
       @mx_servers ||= Resolv::DNS.open do |dns|
         mx_servers = dns.getresources(@address.domain, Resolv::DNS::Resource::IN::MX)
@@ -157,6 +144,10 @@ module EmailAssessor
 
     private
 
+    def domain_in_list?(list)
+      valid? && list.include_any?(domain_tokens)
+    end
+
     def domain_tokens
       @domain_tokens ||= EmailAssessor.tokenize_domain(@address.domain)
     end

data/lib/email_assessor/directory_domain_list.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module EmailAssessor
+  class DirectoryDomainList
+    attr_reader :pathname
+
+    def include_any?(domain_token_set)
+      chars = if @prioritization.present?
+        @prioritization
+      else
+        domain_token_set.indexes
+      end
+
+      chars.any? { |char| domain_list(char).include_any?(domain_token_set) }
+    end
+
+    def sample
+      File.open(Dir.glob(File.join(@pathname, "?.txt")).first, &:readline).chomp
+    end
+
+    private
+
+    def initialize(pathname)
+      prioritization_file_name = File.join(pathname, "_prioritization.txt")
+
+      @prioritization = File.read(prioritization_file_name).split("").freeze if File.exist?(prioritization_file_name)
+      @pathname = pathname
+      @file_map = {
+        # {first_char} => FileDomainList | nil
+      }
+    end
+
+    def domain_list(char)
+      cached = @file_map[char]
+
+      return cached unless cached.nil?
+
+      file_name = File.join(@pathname, "#{char}.txt")
+
+      domain_list = if File.file?(file_name)
+        FileDomainList.new(file_name)
+      else
+        EmptyDomainList.instance
+      end
+
+      @file_map[char] = domain_list
+
+      domain_list
+    end
+  end
+end

data/lib/email_assessor/domain_token_set.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module EmailAssessor
+  class DomainTokenSet
+    class << self
+      def parse(domain)
+        parts = domain.downcase.split(".")
+        indexed_tokens = {
+          # {first_char} => { {segment} => nil }
+        }
+
+        parts.length.times do
+          segment = parts.join(".").freeze
+
+          (indexed_tokens[segment.chr] ||= Hash.new)[segment] = nil
+
+          parts.shift
+        end
+
+        indexed_tokens.each_value(&:freeze)
+        indexed_tokens.freeze
+
+        new(indexed_tokens)
+      end
+    end
+
+    def include?(domain)
+      tokens_of_char = @indexed_tokens[domain.chr]
+
+      return false if tokens_of_char.nil?
+
+      tokens_of_char.key?(domain)
+    end
+
+    def indexes
+      @indexes ||= @indexed_tokens.keys
+    end
+
+    private
+
+    def initialize(indexed_tokens)
+      @indexed_tokens = indexed_tokens
+      @indexes = nil # Shape friendliness
+    end
+  end
+end

data/lib/email_assessor/email_validator.rb

@@ -58,21 +58,5 @@ class EmailValidator < ActiveModel::EachValidator
     if options[:educational]
       error(record, attribute, error_type(:educational, options)) && return if address.educational?
     end
-
-    # if options[:domain_not_in]
-    #   matched_blocklist = options[:domain_not_in].select do |entry|
-    #     unless entry.key?(:blocklist)
-    #       fail "domain_not_in entries must be in format { blocklist: \"filename\"[, message: symbol|string] }"
-    #     end
-
-    #     next unless address.domain_in_blocklist?(entry[:blocklist])
-
-    #     error(record, attribute, entry[:message])
-
-    #     true
-    #   end
-
-    #   return if matched_blocklist
-    # end
   end
 end

data/lib/email_assessor/empty_domain_list.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module EmailAssessor
+  class EmptyDomainList
+    class << self
+      def instance
+        @instance ||= new
+      end
+    end
+
+    def include_any?(*)
+      false
+    end
+
+    def sample
+      nil
+    end
+
+    def pathname
+      nil
+    end
+  end
+end

data/lib/email_assessor/file_domain_list.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module EmailAssessor
+  class FileDomainList
+    attr_reader :pathname
+
+    def include_any?(domain_token_set)
+      File.foreach(@pathname, chomp: true).any? do |domain|
+        domain_token_set.include?(domain)
+      end
+    end
+
+    def sample
+      File.open(@pathname, &:readline).chomp
+    end
+
+    private
+
+    def initialize(pathname)
+      @pathname = pathname
+    end
+  end
+end

data/lib/email_assessor/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module EmailAssessor
-  VERSION = "0.9.1"
+  VERSION = "1.0.0"
 end

data/lib/email_assessor.rb

@@ -1,41 +1,75 @@
 # frozen_string_literal: true
+
 require "email_assessor/email_validator"
+require "email_assessor/directory_domain_list"
+require "email_assessor/file_domain_list"
+require "email_assessor/empty_domain_list"
+require "email_assessor/domain_token_set"
 
 module EmailAssessor
-  DISPOSABLE_DOMAINS_FILE_NAME = File.expand_path("../../vendor/disposable_domains.txt", __FILE__)
-  FASTPASS_DOMAINS_FILE_NAME = File.expand_path("../../vendor/fastpass_domains.txt", __FILE__)
-  EDUCATIONAL_DOMAINS_FILE_NAME = File.expand_path("../../vendor/educational_domains.txt", __FILE__)
-  BLACKLISTED_DOMAINS_FILE_NAME = File.expand_path("vendor/blacklisted_domains.txt")
+  class << self
+    def tokenize_domain(domain)
+      EmailAssessor::DomainTokenSet.parse(domain)
+    end
 
-  def self.domain_is_disposable?(domain)
-    domain_in_file?(domain, DISPOSABLE_DOMAINS_FILE_NAME)
-  end
+    def [](pathname)
+      @domain_list_cache ||= {
+        # {pathname} => {domain list}
+      }
 
-  def self.domain_is_blacklisted?(domain)
-    domain_in_file?(domain, BLACKLISTED_DOMAINS_FILE_NAME)
-  end
+      return @domain_list_cache[pathname] if @domain_list_cache.key?(pathname)
 
-  def self.domain_in_file?(domain, file_name)
-    any_token_in_file?(tokenize_domain(domain), file_name)
-  end
+      list = if File.directory?(pathname)
+        DirectoryDomainList.new(pathname)
+      elsif File.file?(pathname)
+        FileDomainList.new(pathname)
+      end
 
-  def self.any_token_in_file?(domain_tokens, file_name)
-    file_name ||= ""
+      @domain_list_cache[pathname] = list
 
-    File.foreach(file_name, chomp: true).any? do |line|
-      domain_tokens.key?(line)
+      list
     end
-  end
 
-  def self.tokenize_domain(domain)
-    parts = domain.downcase.split(".")
-    tokens = {}
+    def disposable_domains
+      @disposable_domains ||= default_domain_list("disposable_domains")
+    end
+
+    def disposable_domains=(pathname)
+      @disposable_domains = self[pathname]
+    end
+
+    def blacklisted_domains
+      @blacklisted_domains ||= default_domain_list("blacklisted_domains")
+    end
 
-    parts.length.times do
-      tokens[parts.join(".")] = nil
-      parts.shift
+    def blacklisted_domains=(pathname)
+      @blacklisted_domains = self[pathname]
     end
 
-    tokens
+    def educational_domains
+      @educational_domains ||= default_domain_list("educational_domains")
+    end
+
+    def educational_domains=(pathname)
+      @educational_domains = self[pathname]
+    end
+
+    def fastpass_domains
+      @fastpass_domains ||= default_domain_list("fastpass_domains")
+    end
+
+    def fastpass_domains=(pathname)
+      @fastpass_domains = self[pathname]
+    end
+
+    private
+
+    def default_domain_list(category)
+      self[File.expand_path("../../vendor/#{category}.txt", __FILE__)] ||
+        self[File.expand_path("../../vendor/#{category}", __FILE__)] ||
+        self[File.expand_path("../../assets/data/#{category}.txt", __FILE__)] ||
+        self[File.expand_path("../../assets/data/#{category}", __FILE__)] ||
+        EmptyDomainList.instance
+    end
   end
 end