gman 6.0.1 → 7.0.4

data/lib/gman/identifier.rb

@@ -1,9 +1,40 @@
+# frozen_string_literal: true
+
 class Gman
+  # Defines an instance method that delegates to a hash's key
+  #
+  # hash_method -  a symbol representing the instance method to delegate to. The
+  #                instance method should return a hash or respond to #[]
+  # key         - the key to call within the hash
+  # method      - (optional) the instance method the key should be aliased to.
+  #               If not specified, defaults to the hash key
+  # default     - (optional) value to return if value is nil (defaults to nil)
+  #
+  # Returns a symbol representing the instance method
+  def self.def_hash_delegator(hash_method, key, method = nil, default = nil)
+    method ||= key.to_s.downcase.sub(' ', '_')
+    define_method(method) do
+      hash = send(hash_method)
+      if hash.respond_to? :[]
+        hash[key.to_s] || default
+      else
+        default
+      end
+    end
+  end
+
+  def_hash_delegator :dotgov_listing, :Agency
+  def_hash_delegator :dotgov_listing, :Organization
+  def_hash_delegator :dotgov_listing, :City
+  def_hash_delegator :dotgov_listing, :"Domain Type"
+  private :domain_type
+
   def type
-    [:state, :district, :cog, :city, :federal, :county].each do |type|
+    %i[state district cog city federal county].each do |type|
       return type if send "#{type}?"
     end
     return if list_category.nil?
+
     if list_category.include?('usagov')
       :unknown
     else
@@ -14,7 +45,7 @@ class Gman
   def state
     if matches
       matches[4].upcase
-    elsif dotgov_listing
+    elsif dotgov_listing['State']
       dotgov_listing['State']
     elsif list_category
       matches = list_category.match(/usagov([A-Z]{2})/)
@@ -22,27 +53,23 @@ class Gman
     end
   end
 
-  def city
-    dotgov_listing['City'] if dotgov_listing
-  end
-
-  def agency
-    dotgov_listing['Agency'] if federal?
-  end
-
   def dotgov?
     domain.tld == 'gov'
   end
 
   def federal?
-    dotgov_listing && dotgov_listing['Domain Type'] == 'Federal Agency'
+    return false unless dotgov_listing
+
+    domain_type =~ /^Federal Agency/i
   end
 
   def city?
     if matches
-      %w(ci town vil).include?(matches[3])
+      %w[ci town vil].include?(matches[3])
     elsif dotgov_listing
-      dotgov_listing['Domain Type'] == 'City'
+      domain_type == 'City'
+    else
+      false
     end
   end
 
@@ -50,7 +77,9 @@ class Gman
     if matches
       matches[3] == 'co'
     elsif dotgov_listing
-      dotgov_listing['Domain Type'] == 'County'
+      domain_type == 'County'
+    else
+      false
     end
   end
 
@@ -58,40 +87,49 @@ class Gman
     if matches
       matches[1] == 'state'
     elsif dotgov_listing
-      dotgov_listing['Domain Type'] == 'State/Local Govt'
+      domain_type == 'State/Local Govt'
+    else
+      false
     end
   end
 
   def district?
-    matches && matches[1] == 'dst'
+    return false unless matches
+
+    matches[1] == 'dst'
   end
 
   def cog?
-    matches && matches[1] == 'cog'
+    return false unless matches
+
+    matches[1] == 'cog'
   end
 
   private
 
   def list_category
     @list_category ||= begin
-      match = Gman.list.find(domain.to_s)
+      match = Gman.list.public_suffix_list.find(domain.to_s)
       return unless match
-      regex = %r{// ([^\n]+)\n?[^/]*\n#{Regexp.escape(match.name)}\n}im
-      matches = Gman.list_contents.match(regex)
+
+      regex = %r{// ([^\n]+)\n?[^/]*\n#{Regexp.escape(match.value)}\n}im
+      matches = Gman.list.contents.match(regex)
       matches[1] if matches
     end
   end
 
   def matches
     return @matches if defined? @matches
+
     @matches = domain.to_s.match(Locality::REGEX)
   end
 
   def dotgov_listing
     return @dotgov_listing if defined? @dotgov_listing
     return unless dotgov?
+
     @dotgov_listing = Gman.dotgov_list.find do |listing|
-      listing['Domain Name'].casecmp("#{domain.sld}.gov") == 0
+      listing['Domain Name'].casecmp("#{domain.sld}.gov").zero?
     end
   end
 

data/lib/gman/importer.rb

@@ -1,18 +1,21 @@
+# frozen_string_literal: true
+
 # Utility functions for parsing and manipulating public-suffix domain lists
 # Only used in development and not loaded by default
 require 'yaml'
 require 'open-uri'
 require 'resolv'
 require 'logger'
+require 'swot'
 require_relative '../gman'
 require_relative './domain_list'
 
 class Gman
   class Importer
-    attr_accessor :domains
+    attr_accessor :domain_list
 
     # Known false positives from vendored lists
-    BLACKLIST = %w(
+    BLACKLIST = %w[
       business.centurytel.net
       chesnee.net
       citlink.net
@@ -38,31 +41,32 @@ class Gman
       wctc.net
       webconnections.net
       webpages.charter.net
-    ).freeze
+    ].freeze
 
     REGEX_CHECKS = {
-      'home. regex'     => /^home\./,
-      'user. regex'     => /^users?\./,
-      'sites. regex'    => /^sites?\./,
-      'weebly'          => /weebly\.com$/,
-      'wordpress'       => /wordpress\.com$/,
-      'govoffice'       => /govoffice\d?\.com$/,
-      'homestead'       => /homestead\.com$/,
-      'wix.com'         => /wix\.com$/,
-      'blogspot.com'    => /blogspot\.com$/,
-      'tripod.com'      => /tripod\.com$/,
+      'home. regex' => /^home\./,
+      'user. regex' => /^users?\./,
+      'sites. regex' => /^sites?\./,
+      'weebly' => /weebly\.com$/,
+      'wordpress' => /wordpress\.com$/,
+      'govoffice' => /govoffice\d?\.com$/,
+      'homestead' => /homestead\.com$/,
+      'wix.com' => /wix\.com$/,
+      'blogspot.com' => /blogspot\.com$/,
+      'tripod.com' => /tripod\.com$/,
       'squarespace.com' => /squarespace\.com$/,
-      'github.io'       => /github\.io$/,
-      'tumblr'          => /tumblr\.com$/,
-      'locality'        => Gman::Locality::REGEX
+      'github.io' => /github\.io$/,
+      'tumblr' => /tumblr\.com$/,
+      'locality' => Gman::Locality::REGEX,
+      'french edu' => /^ac-.*?\.fr/
     }.freeze
 
     def initialize(domains)
-      @domains = DomainList.new(domains)
+      @domain_list = DomainList.new(data: domains)
     end
 
     def logger
-      @logger ||= Logger.new(STDOUT)
+      @logger ||= Logger.new($stdout)
     end
 
     def normalize_domain(domain)
@@ -71,9 +75,10 @@ class Gman
     end
 
     def valid_domain?(domain, options = {})
-      return false unless ensure_valid(domain)
       return false if !options[:skip_dupe] && !ensure_not_dupe(domain)
+      return false unless ensure_valid(domain)
       return false if !options[:skip_resolve] && !ensure_resolves(domain)
+
       true
     end
 
@@ -81,6 +86,7 @@ class Gman
     # rather than a bool and silence log output
     def reject(domain, reason)
       return reason if ENV['RECONCILING']
+
       logger.info "👎 `#{domain}`: #{reason}"
       false
     end
@@ -89,30 +95,26 @@ class Gman
       @current ||= DomainList.current
     end
 
-    def import(options)
+    def import(options = {})
       logger.info "Current: #{Gman::DomainList.current.count} domains"
-      logger.info "Adding: #{domains.count} domains"
+      logger.info "Adding: #{domain_list.count} domains"
 
       normalize_domains!
       ensure_validity!(options)
 
-      if domains.count == 0
-        logger.info 'Nothing to add. Aborting'
-        exit 0
-      end
-
       add_to_current
       logger.info "New: #{current.count} domains"
     end
 
     def resolver
-      @resolver ||= Resolv::DNS.new(nameserver: ['8.8.8.8', '8.8.4.4'])
+      @resolver ||= Resolv::DNS.new(nameserver: ['1.1.1.1', '8.8.8.8'])
     end
 
     # Verifies that the given domain has an MX record, and thus is valid
     def domain_resolves?(domain)
       domain = Addressable::URI.new(host: domain).normalize.host
       return true if ip?(domain)
+
       returns_record?(domain, 'NS') || returns_record?(domain, 'MX')
     end
 
@@ -120,16 +122,17 @@ class Gman
 
     def ensure_regex(domain)
       REGEX_CHECKS.each do |msg, regex|
-        return reject(domain, msg) if domain =~ regex
+        return reject(domain, msg) if domain&.match?(regex)
       end
       true
     end
 
     def ensure_valid(domain)
       return false if domain.empty?
+
       if BLACKLIST.include?(domain)
         reject(domain, 'blacklist')
-      elsif !PublicSuffix.valid?(".#{domain}")
+      elsif !PublicSuffix.valid?("foo.#{domain}")
         reject(domain, 'invalid')
       elsif Swot.is_academic?(domain)
         reject(domain, 'academic')
@@ -140,11 +143,13 @@ class Gman
 
     def ensure_resolves(domain)
       return reject(domain, 'unresolvable') unless domain_resolves?(domain)
+
       true
     end
 
     def ensure_not_dupe(domain)
       return true unless dupe?(domain)
+
       if current.domains.include?(domain)
         reject(domain, 'duplicate')
       else
@@ -158,22 +163,22 @@ class Gman
     end
 
     def normalize_domains!
-      domains.list.each do |_group, domains|
+      domain_list.to_h.each_value do |domains|
         domains.map! { |domain| normalize_domain(domain) }
         domains.uniq!
       end
     end
 
     def ensure_validity!(options = {})
-      domains.list.each do |_group, domains|
+      domain_list.data.each_value do |domains|
         domains.select! { |domain| valid_domain?(domain, options) }
       end
     end
 
     def add_to_current
-      domains.list.each do |group, domains|
-        current.list[group] ||= []
-        current.list[group].concat domains
+      domain_list.data.each do |group, domains|
+        current.data[group] ||= []
+        current.data[group].concat domains
       end
       current.write
     end
@@ -192,9 +197,3 @@ class Gman
     end
   end
 end
-
-class Gman
-  def self.import(hash, options = {})
-    Gman::Importer.new(hash).import(options)
-  end
-end

data/lib/gman/locality.rb

@@ -1,17 +1,31 @@
+# frozen_string_literal: true
+
 class Gman
+  # Second level .us domains for states and locality
+  # See http://en.wikipedia.org/wiki/.us
+  #
+  # Examples:
+  #  * foo.state.il.us
+  #  * ci.foo.il.us
+  #
+  # Not:
+  #  * state.foo.il.us
+  #  * foo.ci.il.us
+  #  * k12.il.us
+  #  * ci.foo.zx.us
   class Locality
-    AFFINITY_NAMESPACES = %w(state dst cog).freeze
+    AFFINITY_NAMESPACES = %w[state dst cog].freeze
 
-    STATES = %w(
+    STATES = %w[
       ak al ar az ca co ct dc de fl ga hi ia id il in ks ky
       la ma md me mi mn mo ms mt nc nd ne nh nj nm nv ny oh
       ok or pa ri sc sd tn tx um ut va vt wa wi wv wy
-    ).freeze
+    ].freeze
 
-    LOCALITY_DOMAINS = %w(
+    LOCALITY_DOMAINS = %w[
       ci co borough boro city county
       parish town twp vi vil village
-    ).freeze
+    ].freeze
 
     REGEX = /
       (
@@ -19,22 +33,10 @@ class Gman
       |
         (#{Regexp.union(LOCALITY_DOMAINS)})\.[a-z-]+
       )\.(#{Regexp.union(STATES)})\.us
-    /x
-  end
+    /x.freeze
 
-  # Second level .us domains for states and locality
-  # See http://en.wikipedia.org/wiki/.us
-  #
-  # Examples:
-  #  * foo.state.il.us
-  #  * ci.foo.il.us
-  #
-  # Not:
-  #  * state.foo.il.us
-  #  * foo.ci.il.us
-  #  * k12.il.us
-  #  * ci.foo.zx.us
-  def locality?
-    !domain.to_s.match(Locality::REGEX).nil?
+    def self.valid?(domain)
+      !domain.to_s.match(Locality::REGEX).nil?
+    end
   end
 end

data/lib/gman/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class Gman
-  VERSION = '6.0.1'.freeze
+  VERSION = '7.0.4'
 end

data/script/add

@@ -1,4 +1,6 @@
 #! /usr/bin/env ruby
+# frozen_string_literal: true
+
 #
 # Add one or more domains to a given group, running the standard import checks
 #

data/script/alphabetize

@@ -1,4 +1,6 @@
 #! /usr/bin/env ruby
+# frozen_string_literal: true
+
 #
 # Alphabetizes entries in the domains.txt file
 #

data/script/cibuild

@@ -2,7 +2,7 @@
 
 set -ex
 
-bundle exec rake test
+bundle exec rspec
 bundle exec rubocop -D -S -a
 bundle exec script/dedupe
 bundle exec gem build gman.gemspec

data/script/dedupe

@@ -1,4 +1,5 @@
 #! /usr/bin/env ruby
+# frozen_string_literal: true
 
 require 'yaml'
 require 'open-uri'
@@ -12,7 +13,7 @@ puts "Current list contains #{current.count} domains..."
 
 dupe = current.count - current.domains.uniq.count
 puts "Found #{dupe} duplicate domains"
-exit 0 if dupe == 0
+exit 0 if dupe.zero?
 
 dupes = current.domains.select { |domain| current.domains.count(domain) > 1 }