gman 5.0.9 → 6.0.0

data/test/test_gman_bin.rb

@@ -1,81 +1,75 @@
-require_relative "helper"
+require_relative 'helper'
 
 class TestGmanBin < Minitest::Test
-
   def setup
-    @output, @status = test_bin("whitehouse.gov")
+    @output, @status = test_bin('whitehouse.gov')
   end
 
-  should "parse the domain" do
-    output, status = test_bin("bar.gov")
-    assert_match /Domain  : bar.gov/, output
+  should 'parse the domain' do
+    output, = test_bin('bar.gov')
+    assert_match(/Domain  : bar.gov/, output)
 
-    output, status = test_bin("foo@bar.gov")
-    assert_match /Domain  : bar.gov/, output
+    output, = test_bin('foo@bar.gov')
+    assert_match(/Domain  : bar.gov/, output)
 
-    output, status = test_bin("http://bar.gov/foo")
-    assert_match /Domain  : bar.gov/, output
+    output, = test_bin('http://bar.gov/foo')
+    assert_match(/Domain  : bar.gov/, output)
   end
 
-  should "err on invalid domains" do
-    output, status = test_bin("foo.invalid")
+  should 'err on invalid domains' do
+    output, status = test_bin('foo.invalid')
     assert_equal 1, status.exitstatus
-    assert_match /Invalid domain/, output
+    assert_match(/Invalid domain/, output)
   end
 
-  should "err on non-government domains" do
-    output, status = test_bin("github.com")
+  should 'err on non-government domains' do
+    output, status = test_bin('github.com')
     assert_equal 1, status.exitstatus
-    assert_match /Not a government domain/, output
+    assert_match(/Not a government domain/, output)
   end
 
-  should "know the type" do
-    assert_match /federal/, @output
+  should 'know the type' do
+    assert_match(/federal/, @output)
     assert_equal 0, @status.exitstatus
   end
 
-  should "know the agency" do
-    assert_match /Executive Office of the President/, @output
+  should 'know the agency' do
+    assert_match(/Executive Office of the President/, @output)
     assert_equal 0, @status.exitstatus
   end
 
-  should "know the country" do
-    assert_match /United States/, @output
+  should 'know the country' do
+    assert_match(/United States/, @output)
     assert_equal 0, @status.exitstatus
   end
 
-  should "know the city" do
-    assert_match /Washington/, @output
+  should 'know the city' do
+    assert_match(/Washington/, @output)
     assert_equal 0, @status.exitstatus
   end
 
-  should "know the state" do
-    assert_match /DC/, @output
+  should 'know the state' do
+    assert_match(/DC/, @output)
     assert_equal 0, @status.exitstatus
   end
 
-  should "allow you to disable colorization" do
-    output, status = test_bin("whitehouse.gov", "--no-color")
-    refute_match /\e\[32m/, output
+  should 'allow you to disable colorization' do
+    output, = test_bin('whitehouse.gov', '--no-color')
+    refute_match(/\e\[32m/, output)
   end
 
-  should "color by default" do
-    assert_match /\e\[32m/, @output
+  should 'color by default' do
+    assert_match(/\e\[32m/, @output)
   end
 
-  should "show help text" do
-    output, status = test_bin
-    assert_match /Usage/i, output
-
-    output, status = test_bin("")
-    assert_match /Usage/i, output
+  should 'show help text' do
+    output, = test_bin
+    assert_match(/Usage/i, output)
 
-    output, status = test_bin("--no-color")
-    assert_match /Usage/i, output
-  end
+    output, = test_bin('')
+    assert_match(/Usage/i, output)
 
-  should "know if a country is sanctioned" do
-    output, status = test_bin "kim@pyongyang.gov.kp"
-    assert_match /SANCTIONED/, output
+    output, = test_bin('--no-color')
+    assert_match(/Usage/i, output)
   end
 end

data/test/test_gman_country_codes.rb

@@ -2,13 +2,17 @@ require File.join(File.dirname(__FILE__), 'helper')
 
 class TestGmanCountryCodes < Minitest::Test
   should "determine a domain's country" do
-    assert_equal "United States of America", Gman.new("whitehouse.gov").country.name
-    assert_equal "United States of America", Gman.new("army.mil").country.name
-    assert_equal "United Kingdom of Great Britain and Northern Ireland", Gman.new("foo.gov.uk").country.name
-    assert_equal "Canada", Gman.new("foo.gc.ca").country.name
+    name = Gman.new('whitehouse.gov').country.name
+    assert_equal 'United States of America', name
+
+    name = Gman.new('foo.gov.uk').country.name
+    assert_equal 'United Kingdom of Great Britain and Northern Ireland', name
+
+    assert_equal 'United States of America', Gman.new('army.mil').country.name
+    assert_equal 'Canada', Gman.new('foo.gc.ca').country.name
   end
 
-  should "not err out on an unknown country code" do
-    assert_equal nil, Gman.new("foo.eu").country
+  should 'not err out on an unknown country code' do
+    assert_equal nil, Gman.new('foo.eu').country
   end
 end

data/test/test_gman_domains.rb

@@ -1,28 +1,33 @@
 require File.join(File.dirname(__FILE__), 'helper')
 
-class TestDomains < Minitest::Test
-
-  WHITELIST = [ "non-us gov", "non-us mil", "US Federal"]
+class TestGmanDomains < Minitest::Test
+  WHITELIST = ['non-us gov', 'non-us mil', 'US Federal'].freeze
 
   def resolve_domains?
-    ENV["GMAN_RESOLVE_DOMAINS"] == "true"
+    ENV['GMAN_RESOLVE_DOMAINS'] == 'true'
   end
 
-  should "only contains valid domains" do
+  should 'only contains valid domains' do
     importer = Gman::Importer.new({})
     if resolve_domains?
-      importer.logger.info "Validating that all domains resolve. This may take a while..."
+      importer.logger.info <<-MSG
+        Validating that all domains resolve. This may take a while...
+      MSG
     else
-      importer.logger.info "Skipping domain resolution. Run `GMAN_RESOLVE_DOMAINS=true rake test` to validate that domains resolve."
+      importer.logger.info 'Skipping domain resolution.' \
+        'Run `GMAN_RESOLVE_DOMAINS=true rake test` '\
+        'to validate that domains resolve.'
     end
 
     invalid = []
-    Parallel.each(Gman::DomainList.current.list, :in_threads => 2) do |group, domains|
+    list = Gman::DomainList.current.list
+    Parallel.each(list, in_threads: 2) do |group, domains|
       next if WHITELIST.include?(group)
       invalid.push domains.reject { |domain|
-        importer.valid_domain?(domain, :skip_dupe => true, :skip_resolve => !resolve_domains?)
+        options = { skip_dupe: true, skip_resolve: !resolve_domains? }
+        importer.valid_domain?(domain, options)
       }
     end
-    assert_equal [], invalid.flatten.reject { |e| e.empty? }
+    assert_equal [], invalid.flatten.reject(&:empty?)
   end
 end

data/test/test_gman_filter.rb

@@ -2,18 +2,16 @@ HERE = File.dirname(__FILE__)
 require File.join(HERE, 'helper')
 
 class TestGmanFilter < Minitest::Test
+  txt_path = fixture_path 'obama.txt'
+  exec_path = bin_path 'gman_filter'
 
-  txt_path = File.join(HERE, "obama.txt")
-  exec_path = File.join(HERE, "..", "bin", "gman_filter")
-
-  should "remove non-gov/mil addresses" do
-    filtered = `#{exec_path} < #{txt_path}`
+  should 'remove non-gov/mil addresses' do
+    output, _status = Open3.capture2e('bundle', 'exec', exec_path, txt_path)
     expected = %w(
       mr.senator@obama.senate.gov
       president@whitehouse.gov
       commander.in.chief@us.army.mil
     ).join("\n") + "\n"
-    assert_equal filtered, expected
+    assert_equal output, expected
   end
-
 end

data/test/test_gman_identifier.rb

@@ -1,16 +1,16 @@
 require File.join(File.dirname(__FILE__), 'helper')
 
 class TestGmanIdentifier < Minitest::Test
-  should "Parse the dotgov list" do
+  should 'Parse the dotgov list' do
     assert Gman.dotgov_list
     assert_equal CSV::Table, Gman.dotgov_list.class
     assert_equal CSV::Row, Gman.dotgov_list.first.class
-    assert Gman.dotgov_list.first["Domain Name"]
+    assert Gman.dotgov_list.first['Domain Name']
   end
 
-  context "locality domains" do
-    should "detect state domains" do
-      domain = Gman.new("state.ak.us")
+  context 'locality domains' do
+    should 'detect state domains' do
+      domain = Gman.new('state.ak.us')
       assert domain.state?
 
       refute domain.dotgov?
@@ -19,11 +19,11 @@ class TestGmanIdentifier < Minitest::Test
       refute domain.county?
 
       assert_equal :state, domain.type
-      assert_equal "AK", domain.state
+      assert_equal 'AK', domain.state
     end
 
-    should "detect city domains" do
-      domain = Gman.new("ci.champaign.il.us")
+    should 'detect city domains' do
+      domain = Gman.new('ci.champaign.il.us')
       assert domain.city?
 
       refute domain.dotgov?
@@ -32,13 +32,13 @@ class TestGmanIdentifier < Minitest::Test
       refute domain.county?
 
       assert_equal :city, domain.type
-      assert_equal "IL", domain.state
+      assert_equal 'IL', domain.state
     end
   end
 
-  context "dotgovs" do
-    should "detect federal dotgovs" do
-      domain = Gman.new "whitehouse.gov"
+  context 'dotgovs' do
+    should 'detect federal dotgovs' do
+      domain = Gman.new 'whitehouse.gov'
       assert domain.federal?
       assert domain.dotgov?
 
@@ -47,13 +47,13 @@ class TestGmanIdentifier < Minitest::Test
       refute domain.county?
 
       assert_equal :federal, domain.type
-      assert_equal "DC", domain.state
-      assert_equal "Washington", domain.city
-      assert_equal "Executive Office of the President", domain.agency
+      assert_equal 'DC', domain.state
+      assert_equal 'Washington', domain.city
+      assert_equal 'Executive Office of the President', domain.agency
     end
 
-    should "detect state dotgovs" do
-      domain = Gman.new "illinois.gov"
+    should 'detect state dotgovs' do
+      domain = Gman.new 'illinois.gov'
       assert domain.state?
       assert domain.dotgov?
 
@@ -62,12 +62,12 @@ class TestGmanIdentifier < Minitest::Test
       refute domain.county?
 
       assert_equal :state, domain.type
-      assert_equal "IL", domain.state
-      assert_equal "Springfield", domain.city
+      assert_equal 'IL', domain.state
+      assert_equal 'Springfield', domain.city
     end
 
-    should "detect county dotgovs" do
-      domain = Gman.new "ALLEGHENYCOUNTYPA.GOV"
+    should 'detect county dotgovs' do
+      domain = Gman.new 'ALLEGHENYCOUNTYPA.GOV'
       assert domain.county?
       assert domain.dotgov?
 
@@ -76,30 +76,31 @@ class TestGmanIdentifier < Minitest::Test
       refute domain.state?
 
       assert_equal :county, domain.type
-      assert_equal "PA", domain.state
-      assert_equal "Pittsburgh", domain.city
+      assert_equal 'PA', domain.state
+      assert_equal 'Pittsburgh', domain.city
     end
 
-    should "detect the list category" do
-      assert_equal "US Federal", Gman.new("whitehouse.gov").send("list_category")
+    should 'detect the list category' do
+      category = Gman.new('whitehouse.gov').send('list_category')
+      assert_equal 'US Federal', category
     end
   end
 
-  context "non-dotgov domains" do
+  context 'non-dotgov domains' do
     should "determine a domain's group" do
-      assert_equal "usagovIN", Gman.new("cityofperu.org").send("list_category")
-      assert_equal :unknown, Gman.new("cityofperu.org").type
+      assert_equal 'usagovIN', Gman.new('cityofperu.org').send('list_category')
+      assert_equal :unknown, Gman.new('cityofperu.org').type
 
-      assert_equal "Canada municipal", Gman.new("acme.ca").send("list_category")
-      assert_equal :"Canada municipal", Gman.new("acme.ca").type
+      assert_equal 'Canada municipal', Gman.new('acme.ca').send('list_category')
+      assert_equal :"Canada municipal", Gman.new('acme.ca').type
 
-      assert_equal "Canada federal", Gman.new("canada.ca").send("list_category")
-      assert_equal :"Canada federal", Gman.new("canada.ca").type
+      assert_equal 'Canada federal', Gman.new('canada.ca').send('list_category')
+      assert_equal :"Canada federal", Gman.new('canada.ca').type
     end
 
-    should "detect the state" do
-      assert_equal "OR", Gman.new("ashland.or.us").state
-      refute Gman.new("canada.ca").state
+    should 'detect the state' do
+      assert_equal 'OR', Gman.new('ashland.or.us').state
+      refute Gman.new('canada.ca').state
     end
   end
 end

data/test/test_gman_importer.rb

@@ -0,0 +1,250 @@
+require File.join(File.dirname(__FILE__), 'helper')
+
+class TestGManImporter < Minitest::Test
+  def setup
+    @importer = Gman::Importer.new 'test' => ['example.com']
+    @stdout = StringIO.new
+    @importer.instance_variable_set '@logger', Logger.new(@stdout)
+
+    with_env 'GMAN_STUB_DOMAINS', 'true' do
+      @original_domain_list = File.open(Gman.list_path).read
+    end
+  end
+
+  def teardown
+    with_env 'GMAN_STUB_DOMAINS', 'true' do
+      File.write Gman.list_path, @original_domain_list
+    end
+  end
+
+  should 'init the domain list' do
+    assert_equal Gman::DomainList, @importer.domains.class
+    assert_equal 1, @importer.domains.domains.count
+    assert_equal 'example.com', @importer.domains.domains.first
+  end
+
+  should 'init the logger' do
+    assert_equal Logger, @importer.logger.class
+  end
+
+  should 'return the current domain list' do
+    assert_equal Gman::DomainList, @importer.current.class
+  end
+
+  should 'return the resolver' do
+    assert_equal Resolv::DNS, @importer.resolver.class
+  end
+
+  context 'domain rejection' do
+    should 'return false for a rejected domain' do
+      refute @importer.reject 'example.com', 'reasons'
+    end
+
+    should 'return the reason when asked' do
+      with_env 'RECONCILING', 'true' do
+        assert_equal 'reasons', @importer.reject('example.com', 'reasons')
+      end
+    end
+  end
+
+  context 'manipulating the domain list' do
+    should 'normalize domains within the domain list' do
+      importer = Gman::Importer.new 'test' => ['www.EXAMPLE.com/']
+      importer.send :normalize_domains!
+      assert_equal 'example.com', importer.domains.domains.first
+    end
+
+    should 'remove invalid domains from the domain list' do
+      importer = Gman::Importer.new 'test' => ['foo.github.io', 'example.com']
+      importer.instance_variable_set '@logger', Logger.new(@stdout)
+
+      assert_equal 2, importer.domains.domains.count
+      importer.send :ensure_validity!
+      assert_equal 1, importer.domains.domains.count
+    end
+
+    context 'writing the domain list' do
+      should 'add domains to the current domain list' do
+        with_env 'GMAN_STUB_DOMAINS', 'true' do
+          domains = { 'test' => ['example.com'], 'test2' => ['github.com'] }
+          importer = Gman::Importer.new domains
+          importer.send :add_to_current
+          expected = "// test\nexample.com\ngov\n\n// test2\ngithub.com"
+          assert_equal expected, File.open(Gman.list_path).read
+        end
+      end
+
+      should 'import' do
+        with_env 'GMAN_STUB_DOMAINS', 'true' do
+          domains = {
+            'test'  => ['www.example.com', 'goo.github.io'],
+            'test2' => ['github.com', 'www.github.com', 'whitehouse.gov']
+          }
+
+          importer = Gman::Importer.new domains
+          importer.instance_variable_set '@logger', Logger.new(@stdout)
+          importer.import(skip_resolve: true)
+
+          expected = "// test\nexample.com\ngov\n\n// test2\ngithub.com"
+          assert_equal expected, File.open(Gman.list_path).read
+        end
+      end
+    end
+  end
+
+  context 'domain validation' do
+    should 'allow valid domains' do
+      assert @importer.send :ensure_valid, 'whitehouse.gov'
+    end
+
+    should 'reject empty domains' do
+      refute @importer.send :ensure_valid, ''
+    end
+
+    should 'reject blacklisted domains' do
+      refute @importer.send :ensure_valid, 'egovlink.com'
+    end
+
+    should 'reject invalid domains' do
+      refute @importer.send :ensure_valid, 'foo.invalid'
+    end
+
+    should 'reject academic domains' do
+      refute @importer.send :ensure_valid, 'harvard.edu'
+    end
+
+    should "reject regex'd domains" do
+      refute @importer.send :ensure_valid, 'foo.github.io'
+    end
+  end
+
+  context 'duplicate domains' do
+    should 'know a unique domain is not a dupe' do
+      refute @importer.send :dupe?, 'gman.com'
+    end
+
+    should "know when a domain's a dupe" do
+      assert @importer.send :dupe?, 'gov'
+    end
+
+    should "know when a domain's a subdomain of an existing domain" do
+      assert @importer.send :dupe?, 'whitehouse.gov'
+    end
+
+    should 'allow unique domains' do
+      assert @importer.send :ensure_not_dupe, 'gman.com'
+    end
+
+    should 'reject duplicate domains' do
+      refute @importer.send :ensure_not_dupe, 'gov'
+    end
+
+    should 'reject subdomains' do
+      refute @importer.send :ensure_not_dupe, 'whitehouse.gov'
+    end
+  end
+
+  context 'domain resolution' do
+    should 'know if a domain resolves' do
+      assert @importer.domain_resolves?('github.com')
+      assert @importer.send :ensure_resolves, 'github.com'
+    end
+
+    should "know if a domain doesn't resolve" do
+      refute @importer.domain_resolves?('foo.invalid')
+      refute @importer.send :ensure_resolves, 'foo.invalid'
+    end
+
+    should 'know if a domain has an IP' do
+    end
+
+    should 'know if a domain returns a given record' do
+    end
+  end
+
+  context 'regex checks' do
+    should 'pass valid domains' do
+      assert @importer.send :ensure_regex, 'example.com'
+    end
+
+    should 'reject domains that begin with home.' do
+      refute @importer.send :ensure_regex, 'home.example.com'
+    end
+
+    should 'reject domains that begin with user.' do
+      refute @importer.send :ensure_regex, 'user.example.com'
+    end
+
+    should 'reject domains that begin with site.' do
+      refute @importer.send :ensure_regex, 'user.example.com'
+    end
+
+    should 'reject weebly domains' do
+      refute @importer.send :ensure_regex, 'foo.weebly.com'
+    end
+
+    should 'reject wordpress domains' do
+      refute @importer.send :ensure_regex, 'foo.wordpress.com'
+    end
+
+    should 'reject govoffice domains' do
+      refute @importer.send :ensure_regex, 'foo.govoffice.com'
+      refute @importer.send :ensure_regex, 'foo.govoffice1.com'
+    end
+
+    should 'reject homestead domains' do
+      refute @importer.send :ensure_regex, 'foo.homestead.com'
+    end
+
+    should 'reject wix domains' do
+      refute @importer.send :ensure_regex, 'foo.wix.com'
+    end
+
+    should 'reject blogspot domains' do
+      refute @importer.send :ensure_regex, 'foo.blogspot.com'
+    end
+
+    should 'reject tripod domains' do
+      refute @importer.send :ensure_regex, 'foo.tripod.com'
+    end
+
+    should 'reject squarespace domains' do
+      refute @importer.send :ensure_regex, 'foo.squarespace.com'
+    end
+
+    should 'reject github.io domains' do
+      refute @importer.send :ensure_regex, 'foo.github.io'
+    end
+
+    should 'reject locality domains' do
+      refute @importer.send :ensure_regex, 'ci.champaign.il.us'
+    end
+  end
+
+  context 'normalizing domains' do
+    should 'normalize URLs to domains' do
+      expected = 'example.com'
+      assert_equal expected, @importer.normalize_domain('http://example.com')
+    end
+
+    should 'strip WWW' do
+      assert_equal 'example.com', @importer.normalize_domain('www.example.com')
+    end
+
+    should 'remove trailing slashes' do
+      assert_equal 'example.com', @importer.normalize_domain('example.com/')
+    end
+
+    should 'remove paths' do
+      assert_equal 'example.com', @importer.normalize_domain('example.com/foo')
+    end
+
+    should 'remove paths with trailing slashes' do
+      assert_equal 'example.com', @importer.normalize_domain('example.com/foo/')
+    end
+
+    should 'downcase' do
+      assert_equal 'example.com', @importer.normalize_domain('EXAMPLE.com')
+    end
+  end
+end