gman 5.0.9 → 6.0.0

data/gman.gemspec

@@ -1,36 +1,43 @@
-require File.expand_path "./lib/gman/version", File.dirname(__FILE__)
+require File.expand_path './lib/gman/version', File.dirname(__FILE__)
 
 Gem::Specification.new do |s|
-  s.name = "gman"
-  s.summary = "Check if a given domain or email address belong to a governemnt entity"
-  s.description = "A ruby gem to check if the owner of a given email address is working for THE MAN."
+  s.name = 'gman'
+  s.summary = <<-EOF
+    Check if a given domain or email address belong to a governemnt entity
+  EOF
+  s.description = <<-EOF
+    A ruby gem to check if the owner of a given email address is working for
+    THE MAN.
+  EOF
   s.version = Gman::VERSION
-  s.authors = ["Ben Balter"]
-  s.email = "ben.balter@github.com"
-  s.homepage = "https://github.com/benbalter/gman"
-  s.licenses = ["MIT"]
+  s.authors = ['Ben Balter']
+  s.email = 'ben.balter@github.com'
+  s.homepage = 'https://github.com/benbalter/gman'
+  s.licenses = ['MIT']
 
-  s.files                 = `git ls-files`.split("\n")
-  s.test_files            = `git ls-files -- {test,spec,features}/*`.split("\n")
-  s.executables           = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
-  s.require_paths         = ["lib"]
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map do |f|
+    File.basename(f)
+  end
+  s.require_paths = ['lib']
 
-  s.require_paths = ["lib"]
+  s.require_paths = ['lib']
   s.required_ruby_version = '~> 2.0'
 
-  s.add_dependency( "swot", '~> 1.0' )
-  s.add_dependency( "iso_country_codes", "~> 0.6" )
-  s.add_dependency( "naughty_or_nice", "~> 2.0" )
-  s.add_dependency( "colored", "~> 1.2" )
-
-  s.add_development_dependency( "rake", "~> 10.4" )
-  s.add_development_dependency( "shoulda", "~> 3.5" )
-  s.add_development_dependency( "rdoc", "~> 4.2" )
-  s.add_development_dependency( "bundler", "~> 1.10" )
-  s.add_development_dependency( "pry", "~> 0.10" )
-  s.add_development_dependency( "parallel", "~> 1.6" )
-  s.add_development_dependency( "mechanize", "~> 2.7" )
-  s.add_development_dependency( "addressable", "~> 2.3" )
-  s.add_development_dependency( "ruby-prof", "~> 0.15" )
+  s.add_dependency('swot', '~> 1.0')
+  s.add_dependency('iso_country_codes', '~> 0.6')
+  s.add_dependency('naughty_or_nice', '~> 2.0')
+  s.add_dependency('colored', '~> 1.2')
 
+  s.add_development_dependency('rake', '~> 10.4')
+  s.add_development_dependency('shoulda', '~> 3.5')
+  s.add_development_dependency('rdoc', '~> 4.2')
+  s.add_development_dependency('bundler', '~> 1.10')
+  s.add_development_dependency('pry', '~> 0.10')
+  s.add_development_dependency('parallel', '~> 1.6')
+  s.add_development_dependency('mechanize', '~> 2.7')
+  s.add_development_dependency('addressable', '~> 2.3')
+  s.add_development_dependency('ruby-prof', '~> 0.15')
+  s.add_development_dependency('rubocop', '~> 0.37')
 end

data/lib/gman.rb

@@ -6,30 +6,33 @@ require_relative 'gman/version'
 require_relative 'gman/country_codes'
 require_relative 'gman/locality'
 require_relative 'gman/identifier'
-require_relative 'gman/sanctions'
 
 class Gman
-
   include NaughtyOrNice
 
   class << self
     # returns an instance of our custom public suffix list
-    # list behaves like PublicSuffix::List but is limited to our whitelisted domains
+    # list behaves like PublicSuffix::List
+    # but is limited to our whitelisted domains
     def list
-      @@list ||= PublicSuffix::List::parse(list_contents)
+      @list ||= PublicSuffix::List.parse(list_contents)
     end
 
     def config_path
-      File.join(File.dirname(__FILE__), "../config")
+      File.expand_path '../config', File.dirname(__FILE__)
     end
 
     # Returns the absolute path to the domain list
     def list_path
-      File.join(config_path,"domains.txt")
+      if ENV['GMAN_STUB_DOMAINS']
+        File.expand_path '../test/fixtures/domains.txt', File.dirname(__FILE__)
+      else
+        File.expand_path 'domains.txt', config_path
+      end
     end
 
     def list_contents
-      @@list_contents ||= File.new(list_path, "r:utf-8").read
+      @list_contents ||= File.new(list_path, 'r:utf-8').read
     end
   end
 
@@ -37,25 +40,28 @@ class Gman
   #
   # Returns boolean true if a government domain
   def valid?
-    return @valid if defined? @valid
-
-    @valid = begin
-      # Ensure it's a valid domain
-      return false unless domain && domain.valid?
+    @valid ||= begin
+      return false unless valid_domain?
+      return false if academic?
+      locality? || public_suffix_valid?
+    end
+  end
 
-      # Ensure non-edu
-      return false if Swot::is_academic?(domain)
+  private
 
-      # Check for locality by regex
-      return true if locality?
+  def valid_domain?
+    domain && domain.valid? && !academic?
+  end
 
-      # check using public suffix's standard logic
-      rule = Gman.list.find(to_s)
+  def academic?
+    domain && Swot.is_academic?(domain)
+  end
 
-      # domain is on the domain list and
-      # domain is not explicitly blacklisted and
-      # domain matches a standard public suffix list rule
-      !rule.nil? && rule.type != :exception && rule.allow?(".#{domain}")
-    end
+  # domain is on the domain list and
+  # domain is not explicitly blacklisted and
+  # domain matches a standard public suffix list rule
+  def public_suffix_valid?
+    rule = Gman.list.find(to_s)
+    !rule.nil? && rule.type != :exception && rule.allow?(".#{domain}")
   end
 end

data/lib/gman/country_codes.rb

@@ -1,21 +1,20 @@
 class Gman
-
   # Map last part of TLD to alpha2 country code
   ALPHA2_MAP = {
-    :ac     => 'sh',
-    :uk     => 'gb',
-    :su     => 'ru',
-    :tp     => 'tl',
-    :yu     => 'rs',
-    :gov    => "us",
-    :mil    => "us",
-    :org    => "us",
-    :com    => "us",
-    :net    => "us",
-    :edu    => "us",
-    :travel => "us",
-    :info   => "us"
-  }
+    ac:     'sh',
+    uk:     'gb',
+    su:     'ru',
+    tp:     'tl',
+    yu:     'rs',
+    gov:    'us',
+    mil:    'us',
+    org:    'us',
+    com:    'us',
+    net:    'us',
+    edu:    'us',
+    travel: 'us',
+    info:   'us'
+  }.freeze
 
   # Returns the two character alpha county code represented by the domain
   #

data/lib/gman/domain_list.rb

@@ -1,13 +1,12 @@
 class Gman
   class DomainList
-
     attr_accessor :list
-    alias_method :to_h, :list
+    alias to_h list
 
-    COMMENT_REGEX = /\/\/[\/\s]*(.*)$/i
+    COMMENT_REGEX = %r{//[/\s]*(.*)$}i
 
     def initialize(list)
-      @list = list.reject { |group, domains| domains.compact.empty? }
+      @list = list.reject { |_group, domains| domains.compact.empty? }
     end
 
     def groups
@@ -15,7 +14,7 @@ class Gman
     end
 
     def domains
-      list.values.flatten.sort.uniq
+      list.values.flatten.compact.sort.uniq
     end
 
     def count
@@ -23,18 +22,18 @@ class Gman
     end
 
     def alphabetize
-      @list = @list.sort_by { |k,v| k.downcase }.to_h
-      @list.each { |group, domains| domains.sort!.uniq! }
+      @list = @list.sort_by { |k, _v| k.downcase }.to_h
+      @list.each { |_group, domains| domains.sort!.uniq! }
     end
 
     def write
+      alphabetize
       File.write(Gman.list_path, to_public_suffix)
     end
 
     def to_public_suffix
-      current_group = ""
-      output = ""
-      list.sort_by { |group, domains| group.downcase }.each do |group, domains|
+      current_group = output = ''
+      list.sort_by { |group, _domains| group.downcase }.each do |group, domains|
         if group != current_group
           output << "\n\n" unless current_group.empty? # first entry
           output << "// #{group}\n"
@@ -46,7 +45,7 @@ class Gman
     end
 
     def self.current
-      current = File.open(Gman::list_path).read
+      current = File.open(Gman.list_path).read
       DomainList.from_public_suffix(current)
     end
 
@@ -56,23 +55,33 @@ class Gman
       DomainList.new(hash)
     end
 
-    private
+    def parent_domain(domain)
+      domains.find { |c| domain =~ /\.#{Regexp.escape(c)}$/ }
+    end
+
+    class << self
+      private
 
-    # Given an array of comments/domains in public suffix format
-    # Converts to a hash in the form of :group => [domain1, domain2...]
-    def self.array_to_hash(domains)
-      group = ""
-      domain_hash = {}
-      domains.each do |line|
-        next if line.empty?
-        if match = COMMENT_REGEX.match(line)
-          group = match[1]
-        else
-          domain_hash[group] = [] if domain_hash[group].nil?
-          domain_hash[group].push line.downcase
+      # Given an array of comments/domains in public suffix format
+      # Converts to a hash in the form of :group => [domain1, domain2...]
+      def array_to_hash(domains)
+        domain_hash = {}
+        group = ''
+        domains.each do |line|
+          if line =~ COMMENT_REGEX
+            group = COMMENT_REGEX.match(line)[1]
+          else
+            safe_push(domain_hash, group, line.downcase)
+          end
         end
+        domain_hash
+      end
+
+      def safe_push(hash, key, value)
+        return if value.empty?
+        hash[key] ||= []
+        hash[key].push value
       end
-      domain_hash
     end
   end
 end

data/lib/gman/identifier.rb

@@ -1,21 +1,10 @@
 class Gman
-
   def type
-    if state?
-      :state
-    elsif district?
-      :district
-    elsif cog?
-      :cog
-    elsif city?
-      :city
-    elsif federal?
-      :federal
-    elsif county?
-      :county
-    elsif list_category.nil?
-      nil
-    elsif list_category.include?("usagov")
+    [:state, :district, :cog, :city, :federal, :county].each do |type|
+      return type if send "#{type}?"
+    end
+    return if list_category.nil?
+    if list_category.include?('usagov')
       :unknown
     else
       list_category.to_sym
@@ -26,7 +15,7 @@ class Gman
     if matches
       matches[4].upcase
     elsif dotgov_listing
-      dotgov_listing["State"]
+      dotgov_listing['State']
     elsif list_category
       matches = list_category.match(/usagov([A-Z]{2})/)
       matches[1] if matches
@@ -34,80 +23,87 @@ class Gman
   end
 
   def city
-    dotgov_listing["City"] if dotgov_listing
+    dotgov_listing['City'] if dotgov_listing
   end
 
   def agency
-    dotgov_listing["Agency"] if federal?
+    dotgov_listing['Agency'] if federal?
   end
 
   def dotgov?
-    domain.tld == "gov"
+    domain.tld == 'gov'
   end
 
   def federal?
-    dotgov_listing && dotgov_listing["Domain Type"] == "Federal Agency"
+    dotgov_listing && dotgov_listing['Domain Type'] == 'Federal Agency'
   end
 
   def city?
     if matches
-      %w[ci town vil].include?(matches[3])
+      %w(ci town vil).include?(matches[3])
     elsif dotgov_listing
-      dotgov_listing["Domain Type"] == "City"
+      dotgov_listing['Domain Type'] == 'City'
     end
   end
 
   def county?
     if matches
-      matches[3] == "co"
+      matches[3] == 'co'
     elsif dotgov_listing
-      dotgov_listing["Domain Type"] == "County"
+      dotgov_listing['Domain Type'] == 'County'
     end
   end
 
   def state?
     if matches
-      matches[1] == "state"
+      matches[1] == 'state'
     elsif dotgov_listing
-      dotgov_listing["Domain Type"] == "State/Local Govt"
+      dotgov_listing['Domain Type'] == 'State/Local Govt'
     end
   end
 
   def district?
-    !!matches && matches[1] == "dst"
+    matches && matches[1] == 'dst'
   end
 
   def cog?
-    !!matches && matches[1] == "cog"
+    matches && matches[1] == 'cog'
   end
 
   private
 
   def list_category
     @list_category ||= begin
-      if match = Gman.list.find(domain.to_s)
-        regex = Regexp.new "\/\/ ([^\\n]+)\\n?[^\/\/]*\\n#{Regexp.escape(match.name)}\\n", "im"
-        matches = Gman.list_contents.match(regex)
-        matches[1] if matches
-      end
+      match = Gman.list.find(domain.to_s)
+      return unless match
+      regex = %r{// ([^\n]+)\n?[^/]*\n#{Regexp.escape(match.name)}\n}im
+      matches = Gman.list_contents.match(regex)
+      matches[1] if matches
     end
   end
 
   def matches
     return @matches if defined? @matches
-    @matches = domain.to_s.match(LOCALITY_REGEX)
+    @matches = domain.to_s.match(Locality::REGEX)
   end
 
-  def self.dotgov_list_path
-    File.join Gman.config_path, "vendor/dotgovs.csv"
+  def dotgov_listing
+    return @dotgov_listing if defined? @dotgov_listing
+    return unless dotgov?
+    @dotgov_listing = Gman.dotgov_list.find do |listing|
+      listing['Domain Name'].casecmp("#{domain.sld}.gov") == 0
+    end
   end
 
-  def self.dotgov_list
-    @@dotgov_list ||= CSV.read(dotgov_list_path, :headers => true)
-  end
+  class << self
+    def dotgov_list
+      @dotgov_list ||= CSV.read(dotgov_list_path, headers: true)
+    end
 
-  def dotgov_listing
-    return @dotgov_listing if defined? @dotgov_listing
-    @dotgov_listing = Gman.dotgov_list.find { |d| d["Domain Name"].downcase == "#{domain.sld}.gov" } if dotgov?
+    private
+
+    def dotgov_list_path
+      File.join Gman.config_path, 'vendor/dotgovs.csv'
+    end
   end
 end

data/lib/gman/importer.rb

@@ -1,4 +1,4 @@
-# Utility functions for parsing and manipulating public-suffix formatted domain lists
+# Utility functions for parsing and manipulating public-suffix domain lists
 # Only used in development and not loaded by default
 require 'yaml'
 require 'open-uri'
@@ -9,11 +9,10 @@ require_relative './domain_list'
 
 class Gman
   class Importer
-
     attr_accessor :domains
 
     # Known false positives from vendored lists
-    BLACKLIST = %w[
+    BLACKLIST = %w(
       business.centurytel.net
       chesnee.net
       citlink.net
@@ -39,7 +38,24 @@ class Gman
       wctc.net
       webconnections.net
       webpages.charter.net
-    ]
+    ).freeze
+
+    REGEX_CHECKS = {
+      'home. regex'     => /^home\./,
+      'user. regex'     => /^users?\./,
+      'sites. regex'    => /^sites?\./,
+      'weebly'          => /weebly\.com$/,
+      'wordpress'       => /wordpress\.com$/,
+      'govoffice'       => /govoffice\d?\.com$/,
+      'homestead'       => /homestead\.com$/,
+      'wix.com'         => /wix\.com$/,
+      'blogspot.com'    => /blogspot\.com$/,
+      'tripod.com'      => /tripod\.com$/,
+      'squarespace.com' => /squarespace\.com$/,
+      'github.io'       => /github\.io$/,
+      'tumblr'          => /tumblr\.com$/,
+      'locality'        => Gman::Locality::REGEX
+    }.freeze
 
     def initialize(domains)
       @domains = DomainList.new(domains)
@@ -50,40 +66,21 @@ class Gman
     end
 
     def normalize_domain(domain)
-      domain.to_s.downcase.strip.gsub(/^www./, "").gsub(/\/$/, "")
+      domain = Gman.new(domain).to_s
+      domain.to_s.downcase.strip.gsub(/^www./, '').gsub(%r{/$}, '')
     end
 
-    def valid_domain?(domain, options={})
-      return false if domain.empty?
-      return reject(domain, "home. regex")     if domain =~ /^home\./
-      return reject(domain, "user. regex")     if domain =~ /^users?\./
-      return reject(domain, "sites. regex")    if domain =~ /^sites?\./
-      return reject(domain, "weebly")          if domain =~ /weebly\.com$/
-      return reject(domain, "wordpress")       if domain =~ /wordpress\.com$/
-      return reject(domain, "govoffice")       if domain =~ /govoffice\d?\.com$/
-      return reject(domain, "homestead")       if domain =~ /homestead\.com$/
-      return reject(domain, "wix.com")         if domain =~ /wix\.com$/
-      return reject(domain, "blogspot.com")    if domain =~ /blogspot\.com$/
-      return reject(domain, "tripod.com")      if domain =~ /tripod\.com$/
-      return reject(domain, "squarespace.com") if domain =~ /squarespace\.com$/
-      return reject(domain, "github.io")       if domain =~ /github\.io$/
-      return reject(domain, "locality")        if domain =~ Gman::LOCALITY_REGEX
-      return reject(domain, "blacklist")       if BLACKLIST.include?(domain)
-      return reject(domain, "duplicate")       if !options[:skip_dupe] && current.domains.include?(domain)
-      return reject(domain, "invalid")         unless PublicSuffix.valid?(".#{domain}")
-      return reject(domain, "academic")        if Swot::is_academic?(domain)
-
-      if !options[:skip_dupe] && subdomain = current.domains.find { |c| domain =~ /\.#{Regexp.escape(c)}$/}
-        return reject(domain, "subdomain of #{subdomain}")
-      end
-
-      return reject(domain, "unresolvable") if !options[:skip_resolve] && !domain_resolves?(domain)
+    def valid_domain?(domain, options = {})
+      return false unless ensure_valid(domain)
+      return false if !options[:skip_dupe] && !ensure_not_dupe(domain)
+      return false if !options[:skip_resolve] && !ensure_resolves(domain)
       true
     end
 
-    # if RECONCILING=true, return the reason, rather than a bool and silence log output
+    # if RECONCILING=true, return the reason,
+    # rather than a bool and silence log output
     def reject(domain, reason)
-      return reason if ENV["RECONCILING"]
+      return reason if ENV['RECONCILING']
       logger.info "👎 `#{domain}`: #{reason}"
       false
     end
@@ -92,59 +89,112 @@ class Gman
       @current ||= DomainList.current
     end
 
-    def import(options={})
+    def import(options)
       logger.info "Current: #{Gman::DomainList.current.count} domains"
       logger.info "Adding: #{domains.count} domains"
 
-      domains.list.each do |group, domains|
-        domains.map!    { |domain| Gman.new(domain).to_s }
-        domains.map!    { |domain| normalize_domain(domain) }
-        domains.select! { |domain| valid_domain?(domain, options) }
-      end
-
-      logger.info "Filtered to: #{domains.count} domains"
+      normalize_domains!
+      ensure_validity!(options)
 
       if domains.count == 0
-        logger.info "Nothing to add. Aborting"
+        logger.info 'Nothing to add. Aborting'
         exit 0
       end
 
-      domains.list.each do |group,domains|
-        current.list[group] = [] if current.list[group].nil?
-        current.list[group].concat domains
-        current.list[group].sort! # Alphabetize
-        current.list[group].uniq! # Ensure uniqueness
+      add_to_current
+      logger.info "New: #{current.count} domains"
+    end
+
+    def resolver
+      @resolver ||= Resolv::DNS.new(nameserver: ['8.8.8.8', '8.8.4.4'])
+    end
+
+    # Verifies that the given domain has an MX record, and thus is valid
+    def domain_resolves?(domain)
+      domain = Addressable::URI.new(host: domain).normalize.host
+      return true if ip?(domain)
+      returns_record?(domain, 'NS') || returns_record?(domain, 'MX')
+    end
+
+    private
+
+    def ensure_regex(domain)
+      REGEX_CHECKS.each do |msg, regex|
+        return reject(domain, msg) if domain =~ regex
       end
+      true
+    end
 
-      logger.info "New: #{current.count} domains"
+    def ensure_valid(domain)
+      return false if domain.empty?
+      if BLACKLIST.include?(domain)
+        reject(domain, 'blacklist')
+      elsif !PublicSuffix.valid?(".#{domain}")
+        reject(domain, 'invalid')
+      elsif Swot.is_academic?(domain)
+        reject(domain, 'academic')
+      else
+        ensure_regex(domain)
+      end
+    end
 
-      logger.info "Writing to disk..."
-      current.write
-      logger.info "Fin."
+    def ensure_resolves(domain)
+      return reject(domain, 'unresolvable') unless domain_resolves?(domain)
+      true
     end
 
-    def resolver
-      @resolver ||= Resolv::DNS.new(:nameserver => ["8.8.8.8","8.8.4.4"])
+    def ensure_not_dupe(domain)
+      return true unless dupe?(domain)
+      if current.domains.include?(domain)
+        reject(domain, 'duplicate')
+      else
+        parent = current.parent_domain(domain)
+        reject(domain, "subdomain of #{parent}")
+      end
+    end
+
+    def dupe?(domain)
+      current.domains.include?(domain) || current.parent_domain(domain)
+    end
+
+    def normalize_domains!
+      domains.list.each do |_group, domains|
+        domains.map! { |domain| normalize_domain(domain) }
+        domains.uniq!
+      end
+    end
+
+    def ensure_validity!(options = {})
+      domains.list.each do |_group, domains|
+        domains.select! { |domain| valid_domain?(domain, options) }
+      end
+    end
+
+    def add_to_current
+      domains.list.each do |group, domains|
+        current.list[group] ||= []
+        current.list[group].concat domains
+      end
+      current.write
     end
 
-    def resolve_without_errors
-      yield
+    def ip?(domain)
+      resolver.getaddress(domain)
     rescue Resolv::ResolvError
       false
     end
 
-    # Verifies that the given domain has an MX record, and thus is valid
-    def domain_resolves?(domain)
-      domain = Addressable::URI.new(:host => domain).normalize.host
-      resolve_without_errors { resolver.getaddress(domain) } ||
-      resolve_without_errors { resolver.getresource(domain, Resolv::DNS::Resource::IN::NS) } ||
-      resolve_without_errors { resolver.getresource(domain, Resolv::DNS::Resource::IN::MX) }
+    def returns_record?(domain, type)
+      type = Object.const_get "Resolv::DNS::Resource::IN::#{type}"
+      resolver.getresource(domain, type)
+    rescue Resolv::ResolvError
+      false
     end
   end
 end
 
 class Gman
-  def self.import(hash, options={})
+  def self.import(hash, options = {})
     Gman::Importer.new(hash).import(options)
   end
 end