gman 6.0.1 → 7.0.0

data/gman.gemspec

@@ -25,11 +25,11 @@ Gem::Specification.new do |s|
   s.require_paths = ['lib']
   s.required_ruby_version = '~> 2.0'
 
-  s.add_dependency('swot', '~> 1.0')
   s.add_dependency('iso_country_codes', '~> 0.6')
   s.add_dependency('naughty_or_nice', '~> 2.0')
   s.add_dependency('colored', '~> 1.2')
 
+  s.add_development_dependency('swot', '~> 1.0')
   s.add_development_dependency('rake', '~> 10.4')
   s.add_development_dependency('shoulda', '~> 3.5')
   s.add_development_dependency('rdoc', '~> 4.2')

data/lib/gman.rb

@@ -1,38 +1,40 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
 require 'naughty_or_nice'
 require 'swot'
 require 'iso_country_codes'
 require 'csv'
 require_relative 'gman/version'
 require_relative 'gman/country_codes'
-require_relative 'gman/locality'
 require_relative 'gman/identifier'
 
 class Gman
   include NaughtyOrNice
 
+  autoload :DomainList, 'gman/domain_list'
+  autoload :Importer,   'gman/importer'
+  autoload :Locality,   'gman/locality'
+
   class << self
-    # returns an instance of our custom public suffix list
-    # list behaves like PublicSuffix::List
-    # but is limited to our whitelisted domains
     def list
-      @list ||= PublicSuffix::List.parse(list_contents)
+      @list ||= DomainList.new(path: list_path)
+    end
+
+    def academic_list
+      @academic_list ||= DomainList.new(path: academic_list_path)
     end
 
     def config_path
-      File.expand_path '../config', File.dirname(__FILE__)
+      @config_path ||= File.expand_path '../config', File.dirname(__FILE__)
     end
 
     # Returns the absolute path to the domain list
     def list_path
-      if ENV['GMAN_STUB_DOMAINS']
-        File.expand_path '../test/fixtures/domains.txt', File.dirname(__FILE__)
-      else
-        File.expand_path 'domains.txt', config_path
-      end
+      File.expand_path 'domains.txt', config_path
     end
 
-    def list_contents
-      @list_contents ||= File.new(list_path, 'r:utf-8').read
+    def academic_list_path
+      File.expand_path 'vendor/academic.txt', config_path
     end
   end
 
@@ -47,21 +49,21 @@ class Gman
     end
   end
 
+  def locality?
+    Locality.valid?(domain)
+  end
+
   private
 
   def valid_domain?
-    domain && domain.valid? && !academic?
+    @valid_domains ||= domain && domain.valid? && !academic?
   end
 
   def academic?
-    domain && Swot.is_academic?(domain)
+    @academic ||= domain && Gman.academic_list.valid?(to_s)
   end
 
-  # domain is on the domain list and
-  # domain is not explicitly blacklisted and
-  # domain matches a standard public suffix list rule
   def public_suffix_valid?
-    rule = Gman.list.find(to_s)
-    !rule.nil? && rule.type != :exception && rule.allow?(".#{domain}")
+    @public_suffix_valid ||= Gman.list.valid?(to_s)
   end
 end

data/lib/gman/domain_list.rb

@@ -1,39 +1,102 @@
 class Gman
   class DomainList
-    attr_accessor :list
-    alias to_h list
-
     COMMENT_REGEX = %r{//[/\s]*(.*)$}i
 
-    def initialize(list)
-      @list = list.reject { |_group, domains| domains.compact.empty? }
+    attr_writer :data, :path, :contents
+
+    class << self
+      # The current, government domain list
+      def current
+        DomainList.new(path: Gman.list_path)
+      end
+
+      def from_file(path)
+        DomainList.new(path: path)
+      end
+
+      def from_hash(hash)
+        DomainList.new(data: hash)
+      end
+
+      def from_public_suffix(string)
+        DomainList.new(contents: string)
+      end
+      alias from_string from_public_suffix
+    end
+
+    def initialize(path: nil, contents: nil, data: nil)
+      @path     = path
+      @contents = contents
+      @data     = data.reject { |_, domains| domains.compact.empty? } if data
+    end
+
+    # Returns the raw content of the domain list as a string
+    def contents
+      @contents ||= if path
+                      File.new(path, 'r:utf-8').read
+                    else
+                      to_s
+                    end
+    end
+
+    # Returns the parsed contents of the domain list as a hash
+    # in the form for group => domains
+    def data
+      @data ||= string_to_hash(contents)
+    end
+    alias to_h data
+
+    # Returns the path to the domain list on disk
+    def path
+      @path ||= Gman.list_path
+    end
+
+    # returns an instance of our custom public suffix list
+    # list behaves like PublicSuffix::List
+    # but is limited to our whitelisted domains
+    def public_suffix_list
+      @public_suffix_list ||= PublicSuffix::List.parse(contents)
+    end
+
+    # domain is on the domain list and
+    # domain is not explicitly blacklisted and
+    # domain matches a standard public suffix list rule
+    def valid?(domain)
+      rule = public_suffix_list.find(domain)
+      !rule.nil? && rule.type != :exception && rule.allow?(".#{domain}")
     end
 
+    # Returns an array of strings representing the list groups
     def groups
-      list.keys
+      data.keys
     end
 
+    # Return an array of strings representing all domains on the list
     def domains
-      list.values.flatten.compact.sort.uniq
+      data.values.flatten.compact.sort.uniq
     end
 
+    # Return the total number of domains in the list
     def count
       domains.count
     end
 
+    # Alphabetize groups and domains within each group
     def alphabetize
-      @list = @list.sort_by { |k, _v| k.downcase }.to_h
-      @list.each { |_group, domains| domains.sort!.uniq! }
+      @data = data.sort_by { |k, _v| k.downcase }.to_h
+      @data.each { |_group, domains| domains.sort!.uniq! }
     end
 
+    # Write the domain list to disk
     def write
       alphabetize
-      File.write(Gman.list_path, to_public_suffix)
+      File.write(path, to_public_suffix)
     end
 
-    def to_public_suffix
+    # The string representation of the domain list, in public suffix format
+    def to_s
       current_group = output = ''
-      list.sort_by { |group, _domains| group.downcase }.each do |group, domains|
+      data.sort_by { |group, _| group.downcase }.each do |group, domains|
         if group != current_group
           output << "\n\n" unless current_group.empty? # first entry
           output << "// #{group}\n"
@@ -43,45 +106,48 @@ class Gman
       end
       output
     end
+    alias to_public_suffix to_s
 
-    def self.current
-      current = File.open(Gman.list_path).read
-      DomainList.from_public_suffix(current)
+    # Given a domain, find any domain on the list that includes that domain
+    # E.g., `fcc.gov` would be the parent of `data.fcc.gov`
+    def parent_domain(domain)
+      domains.find { |c| domain =~ /\.#{Regexp.escape(c)}$/ }
     end
 
-    def self.from_public_suffix(string)
-      string = string.gsub(/\r\n?/, "\n").split("\n")
-      hash = array_to_hash(string)
-      DomainList.new(hash)
+    private
+
+    # Parse a public-suffix formatted string into a hash of groups => [domains]
+    def string_to_hash(string)
+      return unless string
+      lines = string_to_array(string)
+      array_to_hash(lines)
     end
 
-    def parent_domain(domain)
-      domains.find { |c| domain =~ /\.#{Regexp.escape(c)}$/ }
+    def string_to_array(string)
+      string.gsub(/\r\n?/, "\n").split("\n")
     end
 
-    class << self
-      private
-
-      # Given an array of comments/domains in public suffix format
-      # Converts to a hash in the form of :group => [domain1, domain2...]
-      def array_to_hash(domains)
-        domain_hash = {}
-        group = ''
-        domains.each do |line|
-          if line =~ COMMENT_REGEX
-            group = COMMENT_REGEX.match(line)[1]
-          else
-            safe_push(domain_hash, group, line.downcase)
-          end
+    def array_to_hash(lines)
+      domain_hash = {}
+      group = ''
+      lines.each do |line|
+        if line =~ COMMENT_REGEX
+          group = COMMENT_REGEX.match(line)[1]
+        else
+          safe_push(domain_hash, group, line.downcase)
         end
-        domain_hash
       end
+      domain_hash
+    end
 
-      def safe_push(hash, key, value)
-        return if value.empty?
-        hash[key] ||= []
-        hash[key].push value
-      end
+    # Add a value to an array in a hash, creating the array if necessary
+    # hash  - the hash
+    # key   - the key within that hash to add the value to
+    # value - the single value to push into the array at hash[key]
+    def safe_push(hash, key, value)
+      return if value.empty?
+      hash[key] ||= []
+      hash[key].push value
     end
   end
 end

data/lib/gman/identifier.rb

@@ -74,10 +74,10 @@ class Gman
 
   def list_category
     @list_category ||= begin
-      match = Gman.list.find(domain.to_s)
+      match = Gman.list.public_suffix_list.find(domain.to_s)
       return unless match
       regex = %r{// ([^\n]+)\n?[^/]*\n#{Regexp.escape(match.name)}\n}im
-      matches = Gman.list_contents.match(regex)
+      matches = Gman.list.contents.match(regex)
       matches[1] if matches
     end
   end

data/lib/gman/importer.rb

@@ -9,7 +9,7 @@ require_relative './domain_list'
 
 class Gman
   class Importer
-    attr_accessor :domains
+    attr_accessor :domain_list
 
     # Known false positives from vendored lists
     BLACKLIST = %w(
@@ -58,7 +58,7 @@ class Gman
     }.freeze
 
     def initialize(domains)
-      @domains = DomainList.new(domains)
+      @domain_list = DomainList.new(data: domains)
     end
 
     def logger
@@ -71,8 +71,8 @@ class Gman
     end
 
     def valid_domain?(domain, options = {})
-      return false unless ensure_valid(domain)
       return false if !options[:skip_dupe] && !ensure_not_dupe(domain)
+      return false unless ensure_valid(domain)
       return false if !options[:skip_resolve] && !ensure_resolves(domain)
       true
     end
@@ -89,18 +89,13 @@ class Gman
       @current ||= DomainList.current
     end
 
-    def import(options)
+    def import(options = {})
       logger.info "Current: #{Gman::DomainList.current.count} domains"
-      logger.info "Adding: #{domains.count} domains"
+      logger.info "Adding: #{domain_list.count} domains"
 
       normalize_domains!
       ensure_validity!(options)
 
-      if domains.count == 0
-        logger.info 'Nothing to add. Aborting'
-        exit 0
-      end
-
       add_to_current
       logger.info "New: #{current.count} domains"
     end
@@ -158,22 +153,22 @@ class Gman
     end
 
     def normalize_domains!
-      domains.list.each do |_group, domains|
+      domain_list.to_h.each do |_group, domains|
         domains.map! { |domain| normalize_domain(domain) }
         domains.uniq!
       end
     end
 
     def ensure_validity!(options = {})
-      domains.list.each do |_group, domains|
+      domain_list.data.each do |_group, domains|
         domains.select! { |domain| valid_domain?(domain, options) }
       end
     end
 
     def add_to_current
-      domains.list.each do |group, domains|
-        current.list[group] ||= []
-        current.list[group].concat domains
+      domain_list.data.each do |group, domains|
+        current.data[group] ||= []
+        current.data[group].concat domains
       end
       current.write
     end
@@ -192,9 +187,3 @@ class Gman
     end
   end
 end
-
-class Gman
-  def self.import(hash, options = {})
-    Gman::Importer.new(hash).import(options)
-  end
-end

data/lib/gman/locality.rb

@@ -1,4 +1,16 @@
 class Gman
+  # Second level .us domains for states and locality
+  # See http://en.wikipedia.org/wiki/.us
+  #
+  # Examples:
+  #  * foo.state.il.us
+  #  * ci.foo.il.us
+  #
+  # Not:
+  #  * state.foo.il.us
+  #  * foo.ci.il.us
+  #  * k12.il.us
+  #  * ci.foo.zx.us
   class Locality
     AFFINITY_NAMESPACES = %w(state dst cog).freeze
 
@@ -20,21 +32,9 @@ class Gman
         (#{Regexp.union(LOCALITY_DOMAINS)})\.[a-z-]+
       )\.(#{Regexp.union(STATES)})\.us
     /x
-  end
 
-  # Second level .us domains for states and locality
-  # See http://en.wikipedia.org/wiki/.us
-  #
-  # Examples:
-  #  * foo.state.il.us
-  #  * ci.foo.il.us
-  #
-  # Not:
-  #  * state.foo.il.us
-  #  * foo.ci.il.us
-  #  * k12.il.us
-  #  * ci.foo.zx.us
-  def locality?
-    !domain.to_s.match(Locality::REGEX).nil?
+    def self.valid?(domain)
+      !domain.to_s.match(Locality::REGEX).nil?
+    end
   end
 end

data/lib/gman/version.rb

@@ -1,3 +1,3 @@
 class Gman
-  VERSION = '6.0.1'.freeze
+  VERSION = '7.0.0'.freeze
 end

data/script/vendor

@@ -6,7 +6,7 @@ for file in script/vendor-*; do
     echo "*************************************"
     echo "Vendoring $file"
     echo "*************************************"
-    "$file"
+    bundle exec "$file"
   fi
 done
 

data/script/vendor-federal-de

@@ -3,7 +3,6 @@
 require 'csv'
 require 'open-uri'
 require './lib/gman'
-require './lib/gman/importer'
 
 url = 'https://raw.githubusercontent.com/robbi5/german-gov-domains/master/data/domains.csv'
 
@@ -11,4 +10,4 @@ domains = open(url).read.encode('UTF-8')
 domains = CSV.parse(domains, headers: true)
 domains = domains.map { |row| row['Domain Name'] }
 
-Gman.import('German Federal' => domains)
+Gman::Importer.new('German Federal' => domains).import