gman 4.6.5 → 4.7.0

data/script/vendor-us

@@ -3,9 +3,6 @@
 # Vendors the USA.gov-maintained list of US domains into domains.txt
 # Source: https://github.com/GSA-OCSIT/govt-urls
 #
-# Normalizes and cleans inputs, validates domains, rejects academic domains, and
-# sorts, ensures uniqueness, and merges into the existing lib/domains.txt list
-#
 # Usage: script/vendor-us
 #
 # Will automatically fetch latest version of the list and merge
@@ -13,70 +10,13 @@
 #
 # It's also probably a good idea to run `script/ci-build` for good measure
 
-require 'rubygems'
-require 'public_suffix'
-require 'swot'
-require 'yaml'
-require 'open-uri'
-require './lib/gman'
-require './lib/gman/parser'
-
-SOURCE = "https://raw.githubusercontent.com/GSA/govt-urls/master/government-urls.yaml"
-BLACKLIST = ["usagovQUASI", "usagovFED", "usagovPW"]
-domain_hash = {}
-
-domain_hash = YAML.load(open(SOURCE).read)
-puts "found #{domain_hash.map { |group,domains| domains.count }.inject(:+)} domains..."
-
-# Normalize ALL THE THINGS
-domain_hash.each do |group, domains|
-  domains.map! { |domain| domain.strip } # Strip trailing slashes
-  domains.map! { |domain| domain.gsub /\/$/, "" } # Strip trailing slashes
-  domains.map! { |domain| domain.downcase } # make lower case
-  domains.reject! { |domain| domain.empty? }      # Reject empty strings
-end
-
-# filter
-domain_hash.reject! { |group,domain| BLACKLIST.include?(group) } # Group blacklist
-domain_hash.each do |group, domains|
-  puts "Filtering #{group}..."
-  domains.reject! { |domain| domain.match /\// }           # Reject URLs
-  domains.select! { |domain| PublicSuffix.valid?(domain) } # Validate domain
-  domains.reject! { |domain| Swot::is_academic?(domain) }  # Reject academic domains
-end
-puts "Filtered down to #{domain_hash.map { |group,domains| domains.count }.inject(:+)} domains"
-
-# Grab existing list
-current = Gman::Parser.file_to_array( Gman::list_path )
-current_hash = Gman::Parser.array_to_hash(current)
-puts "Current list contains #{current.size} domains... merging"
-
-# Lazy deep merge
-domain_hash.each do |group,domains|
-  current_hash[group] = [] if current_hash[group].nil?
-  current_hash[group].concat domains
-  current_hash[group].sort! # Alphabetize
-  current_hash[group].uniq! # Ensure uniqueness
-end
-
-# Sort by group
-current_hash = current_hash.sort_by { |group, domains| group.downcase }
-
-# PublicSuffix Formatted Output
-current_group = ""
-output = ""
-current_hash.each do |group, domains|
-  if group != current_group
-    output << "\n\n" unless current_group.empty? # first entry
-    output << "// #{group}\n"
-    current_group = group
-  end
-  output << domains.join("\n")
-end
+require './lib/gman/importer'
 
-puts "merged. Writing..."
+source = "https://raw.githubusercontent.com/GSA/govt-urls/master/government-urls.yaml"
+blacklist = ["usagovQUASI", "usagovFED", "usagovPW"]
 
-File.open(Gman.list_path, "w") { |file| file.write output }
+data = open(source).read
+domains = YAML.load(data)
+domains.reject! { |group,domain| blacklist.include?(group) }
 
-result = Gman::Parser.file_to_array( Gman::list_path )
-puts "New list contains #{result.size} domains. Fin."
+Gman.import(domains)

data/test/helper.rb

@@ -16,10 +16,12 @@ require 'shoulda'
 
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
-require 'gman'
+require_relative "../lib/gman"
+require_relative "../lib/gman/domain_list"
+
 require 'net/dns'
 require 'net/dns/resolver'
-require './lib/gman/parser'
+require './lib/gman/importer'
 
 def test_bin(*args)
   output, status = Open3.capture2e("bundle", "exec", "gman", *args)

data/test/test_gman_country_codes.rb

@@ -2,9 +2,9 @@ require File.join(File.dirname(__FILE__), 'helper')
 
 class TestGmanCountryCodes < Minitest::Test
   should "determine a domain's country" do
-    assert_equal "United States", Gman.new("whitehouse.gov").country.name
-    assert_equal "United States", Gman.new("army.mil").country.name
-    assert_equal "United Kingdom", Gman.new("foo.gov.uk").country.name
+    assert_equal "United States of America", Gman.new("whitehouse.gov").country.name
+    assert_equal "United States of America", Gman.new("army.mil").country.name
+    assert_equal "United Kingdom of Great Britain and Northern Ireland", Gman.new("foo.gov.uk").country.name
     assert_equal "Canada", Gman.new("foo.gc.ca").country.name
   end
 end

data/test/test_gman_domains.rb

@@ -0,0 +1,28 @@
+require File.join(File.dirname(__FILE__), 'helper')
+
+class TestDomains < Minitest::Test
+
+  WHITELIST = [ "non-us gov", "non-us mil", "US Federal"]
+
+  def resolve_domains?
+    ENV["GMAN_RESOLVE_DOMAINS"] == "true"
+  end
+
+  should "only contains valid domains" do
+    importer = Gman::Importer.new({})
+    if resolve_domains?
+      importer.logger.info "Validating that all domains resolve. This may take a while..."
+    else
+      importer.logger.info "Skipping domain resolution. Run `GMAN_RESOLVE_DOMAINS=true rake test` to validate that domains resolve."
+    end
+
+    invalid = []
+    Parallel.each(Gman::DomainList.current.list, :in_threads => 2) do |group, domains|
+      next if WHITELIST.include?(group)
+      invalid.push domains.reject { |domain|
+        importer.valid_domain?(domain, :skip_dupe => true, :skip_resolve => !resolve_domains?)
+      }
+    end
+    assert_equal [], invalid.flatten.reject { |e| e.empty? }
+  end
+end

data/test/test_gman_identifier.rb

@@ -98,7 +98,6 @@ class TestGmanIdentifier < Minitest::Test
     end
 
     should "detect the state" do
-      assert_equal "PR", Gman.new("sanjuan.pr").state
       assert_equal "OR", Gman.new("ashland.or.us").state
       refute Gman.new("canada.ca").state
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gman
 version: !ruby/object:Gem::Version
-  version: 4.6.5
+  version: 4.7.0
 platform: ruby
 authors:
 - Ben Balter
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-15 00:00:00.000000000 Z
+date: 2015-06-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: swot
@@ -188,29 +188,30 @@ files:
 - gman.gemspec
 - lib/gman.rb
 - lib/gman/country_codes.rb
+- lib/gman/domain_list.rb
 - lib/gman/identifier.rb
+- lib/gman/importer.rb
 - lib/gman/locality.rb
-- lib/gman/parser.rb
 - lib/gman/sanctions.rb
 - script/alphabetize
-- script/build
 - script/cibuild
 - script/console
 - script/dedupe
 - script/prune
 - script/release
-- script/state-domains
-- script/vendor-de
+- script/vendor-federal-de
 - script/vendor-gov-list
+- script/vendor-municipal-de
 - script/vendor-nl
+- script/vendor-public-suffix
 - script/vendor-se
 - script/vendor-us
 - test/helper.rb
 - test/obama.txt
-- test/test_domains.rb
 - test/test_gman.rb
 - test/test_gman_bin.rb
 - test/test_gman_country_codes.rb
+- test/test_gman_domains.rb
 - test/test_gman_filter.rb
 - test/test_gman_identifier.rb
 - test/test_gman_locality.rb
@@ -242,10 +243,10 @@ summary: Check if a given domain or email address belong to a governemnt entity
 test_files:
 - test/helper.rb
 - test/obama.txt
-- test/test_domains.rb
 - test/test_gman.rb
 - test/test_gman_bin.rb
 - test/test_gman_country_codes.rb
+- test/test_gman_domains.rb
 - test/test_gman_filter.rb
 - test/test_gman_identifier.rb
 - test/test_gman_locality.rb

data/lib/gman/parser.rb

@@ -1,59 +0,0 @@
-# Utility functions for parsing and manipulating public-suffix formatted domain lists
-require 'net/dns'
-require 'net/dns/resolver'
-
-class Gman < NaughtyOrNice
-  class Parser
-
-    COMMENT_REGEX = /\/\/[\/\s]*(.*)$/i
-
-    class << self
-
-      # Given a public-suffix list formatted file
-      # Converts to a hash in the form of :group => [domain1, domain2...]
-      def file_to_hash(file)
-        array_to_hash(file_to_array(file))
-      end
-
-      # Given a public-suffix list formatted file
-      # Convert it into an array of comments and domains representing each line
-      def file_to_array(file)
-        domains = File.open(file).read
-        domains.gsub! /\r\n?/, "\n" # Normalize line endings
-        domains = domains.split("\n")
-      end
-
-      # Given an array of comments/domains in public suffix format
-      # Converts to a hash in the form of :group => [domain1, domain2...]
-      def array_to_hash(domains)
-        group = ""
-        domain_hash = {}
-        domains.each do |line|
-          next if line.empty?
-          if match = COMMENT_REGEX.match(line)
-            group = match[1]
-          else
-            domain_hash[group] = [] if domain_hash[group].nil?
-            domain_hash[group].push line.downcase
-          end
-        end
-        domain_hash
-      end
-
-      def resolver
-        @resolver ||= begin
-          resolver = Net::DNS::Resolver.new
-          resolver.nameservers = ["8.8.8.8","8.8.4.4", "208.67.222.222", "208.67.220.220"]
-          resolver
-        end
-      end
-
-      # Verifies that the given domain has an MX record, and thus is valid
-      def domain_resolves?(domain)
-        resolver.search(domain).header.anCount > 0 ||
-        resolver.search(domain, Net::DNS::NS).header.anCount > 0 ||
-        resolver.search(domain, Net::DNS::MX).header.anCount > 0
-      end
-    end
-  end
-end

data/script/state-domains

@@ -1,38 +0,0 @@
-#! /usr/bin/env ruby
-# Strips domains in the form of e.g., city.<locality>.<state>.us from the domain list
-
-require './lib/gman'
-require './lib/gman/parser'
-
-current = Gman::Parser.file_to_array( Gman::list_path )
-domain_hash = Gman::Parser.array_to_hash(current)
-
-puts "Checking for state gov regex'd domains in the list..."
-puts "Starting with #{current.size} domains..."
-
-domain_hash.each do |group, domains|
-  next unless group =~ /usagov[A-Z]{2}/
-  state = group[-2,2].downcase
-  domain_hash[group].reject! { |d| d =~ Gman::LOCALITY_REGEX }
-  domain_hash[group].uniq!
-  domain_hash[group].sort!
-end
-
-# PublicSuffix Formatted Output
-current_group = ""
-output = ""
-domain_hash.each do |group, domains|
-  if group != current_group
-    output << "\n\n" unless current_group.empty? # first entry
-    output << "// #{group}\n"
-    current_group = group
-  end
-  output << domains.join("\n")
-end
-
-File.open(Gman.list_path, "w") { |file| file.write output }
-
-result = Gman::Parser.file_to_array( Gman::list_path )
-puts "New list contains #{result.size} domains. Fin."
-
-exit 1 if current.size != result.size

data/script/vendor-de

@@ -1,44 +0,0 @@
-#! /usr/bin/env ruby
-
-require 'csv'
-require 'open-uri'
-require './lib/gman'
-require './lib/gman/parser'
-
-source = "http://www.mik.nrw.de/nc/themen-aufgaben/kommunales/kommunale-adressen.html?tx_szkommunaldb_pi1%5Bexport%5D=csv"
-
-csv = open(source).read.force_encoding("iso-8859-1").encode("UTF-8")
-
-# For some reason, the header row is actually the last row
-# Pop the last line off the file and prepend it at the begining
-# So that when we pass it to CSV it detects the headers properly
-lines = csv.split("\n")
-lines.unshift lines.pop
-csv = lines.join("\n")
-
-data = CSV.parse(csv, :headers => true, :col_sep => ";")
-domains = data.map { |row| row["Internet"].to_s.downcase.strip.gsub /^www./, "" }
-
-domains.reject! { |domain| domain.empty? }
-domains.select! { |domain| PublicSuffix.valid?(".#{domain}") } # Validate domain
-domains.reject! { |domain| Swot::is_academic?(domain) }  # Reject academic domains
-
-current = Gman::Parser.file_to_array( Gman::list_path )
-current_hash = Gman::Parser.array_to_hash(current)
-
-current_hash["German Municipalities"] = domains
-current_hash = current_hash.sort_by { |group, domains| group.downcase }
-
-# PublicSuffix Formatted Output
-current_group = ""
-output = ""
-current_hash.each do |group, domains|
-  if group != current_group
-    output << "\n\n" unless current_group.empty? # first entry
-    output << "// #{group}\n"
-    current_group = group
-  end
-  output << domains.join("\n")
-end
-
-File.open(Gman.list_path, "w") { |file| file.write output }

data/test/test_domains.rb

@@ -1,54 +0,0 @@
-require File.join(File.dirname(__FILE__), 'helper')
-
-class TestDomains < Minitest::Test
-
-  WHITELIST = [ "non-us gov", "non-us mil", "US Federal"]
-  DOMAINS = Gman::Parser.file_to_hash(Gman.list_path)
-
-  def whitelisted?(domain)
-    WHITELIST.each do |group|
-      return true if DOMAINS[group].include? domain
-    end
-    false
-  end
-
-  should "only contain resolvable domains" do
-    unresolvables = []
-    Parallel.each(Gman.list, :in_threads => 2) do |entry|
-      next if whitelisted? entry.name
-      resolves = Gman::Parser.domain_resolves?(entry.name)
-      unresolvables.push entry.name unless resolves
-    end
-    assert_equal [], unresolvables
-  end
-
-  should "not contain any educational domains" do
-    Parallel.each(Gman.list, :in_threads => 2) do |entry|
-      assert_equal false, Swot::is_academic?(entry.name), "#{entry.name} is an academic domain"
-    end
-  end
-
-  should "not contain any invalid domains" do
-    Parallel.each(Gman.list, :in_threads => 2) do |entry|
-      assert_equal true, PublicSuffix.valid?("foo.#{entry.name}"), "#{entry.name} is not a valid domain"
-    end
-  end
-
-  should "pass any url on the list" do
-    Parallel.each(Gman.list, :in_threads => 2) do |entry|
-      assert_equal true, Gman.valid?("http://foo.#{entry.name}/bar"), "http://foo.#{entry.name}/bar is not a valid"
-    end
-  end
-
-  should "pass any email on the list" do
-    Parallel.each(Gman.list, :in_threads => 2) do |entry|
-      assert_equal true, Gman.valid?("foo@bar.#{entry.name}"), "foo@bar.#{entry.name} is not a valid"
-    end
-  end
-
-  should "pass any domain on the list" do
-    Parallel.each(Gman.list, :in_threads => 2) do |entry|
-      assert_equal true, Gman.valid?("foo.#{entry.name}"), "foo.#{entry.name} is not a valid domain"
-    end
-  end
-end