gman 7.0.1 → 7.0.2

data/test/test_gman_country_codes.rb

@@ -1,18 +0,0 @@
-require File.join(File.dirname(__FILE__), 'helper')
-
-class TestGmanCountryCodes < Minitest::Test
-  should "determine a domain's country" do
-    name = Gman.new('whitehouse.gov').country.name
-    assert_equal 'United States of America', name
-
-    name = Gman.new('foo.gov.uk').country.name
-    assert_equal 'United Kingdom of Great Britain and Northern Ireland', name
-
-    assert_equal 'United States of America', Gman.new('army.mil').country.name
-    assert_equal 'Canada', Gman.new('foo.gc.ca').country.name
-  end
-
-  should 'not err out on an unknown country code' do
-    assert_equal nil, Gman.new('foo.eu').country
-  end
-end

data/test/test_gman_domain_list.rb

@@ -1,112 +0,0 @@
-require File.join(File.dirname(__FILE__), 'helper')
-
-class TestGmanDomainList < Minitest::Test
-  INIT_TYPES = [:path, :contents, :data].freeze
-
-  def setup
-    @original_domain_list = File.read(stubbed_list_path)
-  end
-
-  def teardown
-    File.write stubbed_list_path, @original_domain_list
-  end
-
-  def domain_list(type)
-    case type
-    when :path
-      Gman::DomainList.new(path: Gman.list_path)
-    when :contents
-      contents = File.read(Gman.list_path)
-      Gman::DomainList.new(contents: contents)
-    when :data
-      data = Gman::DomainList.new(path: Gman.list_path).to_h
-      Gman::DomainList.new(data: data)
-    end
-  end
-
-  INIT_TYPES.each do |type|
-    context "when initalized with #{type}" do
-      should 'store the init vars' do
-        refute domain_list(type).public_send(type).nil?
-      end
-
-      should 'return the domain data' do
-        list = domain_list(type)
-        assert list.data.key? 'Canada federal'
-        assert list.data.any? { |_key, values| values.include? 'gov' }
-      end
-
-      should 'return the list contents' do
-        list = domain_list(type)
-        assert_match(/^gov$/, list.contents)
-      end
-
-      should 'return the list path' do
-        list = domain_list(type)
-        assert_equal list.path, Gman.list_path
-      end
-
-      should 'return the public suffix parsed list' do
-        list = domain_list(type)
-        assert list.public_suffix_list.class == PublicSuffix::List
-      end
-
-      should 'know if a domain is valid' do
-        list = domain_list(type)
-        assert list.valid? 'whitehouse.gov'
-      end
-
-      should 'know if a domain is invalid' do
-        list = domain_list(type)
-        refute list.valid? 'example.com'
-      end
-
-      should 'return the domain groups' do
-        list = domain_list(type)
-        assert list.groups.include?('Canada federal')
-      end
-
-      should 'return the domains' do
-        list = domain_list(type)
-        assert list.domains.include?('gov')
-      end
-
-      should 'return the domain count' do
-        list = domain_list(type)
-        assert list.count.is_a?(Integer)
-        assert list.count > 100
-      end
-
-      should 'alphabetize the list' do
-        list = domain_list(type)
-        list.data['Canada municipal'].shuffle!
-        assert list.data['Canada municipal'].first != '100milehouse.com'
-        list.alphabetize
-        assert list.data['Canada municipal'].first == '100milehouse.com'
-      end
-
-      should 'write the list' do
-        list = domain_list(type)
-        list.instance_variable_set('@path', stubbed_list_path)
-        list.data = { 'foo' => ['bar.gov', 'baz.net'] }
-        list.write
-        contents = File.read(stubbed_list_path)
-        assert_match %r{^// foo$}, contents
-        expected = "// foo\nbar.gov\nbaz.net"
-        assert contents.include?(expected)
-      end
-
-      should 'output the list in public_suffix format' do
-        list = domain_list(type)
-        string = list.to_s
-        assert_match %r{^// Canada federal$}, string
-        assert string.include? "// Canada federal\ncanada\.ca\n"
-      end
-
-      should "find a domain's parent" do
-        list = domain_list(type)
-        assert_equal 'gov.uk', list.parent_domain('foo.gov.uk')
-      end
-    end
-  end
-end

data/test/test_gman_domains.rb

@@ -1,32 +0,0 @@
-require File.join(File.dirname(__FILE__), 'helper')
-
-class TestGmanDomains < Minitest::Test
-  WHITELIST = ['non-us gov', 'non-us mil', 'US Federal'].freeze
-
-  def resolve_domains?
-    ENV['GMAN_RESOLVE_DOMAINS'] == 'true'
-  end
-
-  should 'only contains valid domains' do
-    importer = Gman::Importer.new({})
-    if resolve_domains?
-      importer.logger.info <<-MSG
-        Validating that all domains resolve. This may take a while...
-      MSG
-    else
-      importer.logger.info 'Skipping domain resolution.' \
-        'Run `GMAN_RESOLVE_DOMAINS=true rake test` '\
-        'to validate that domains resolve.'
-    end
-
-    invalid = []
-    options = { skip_dupe: true, skip_resolve: !resolve_domains? }
-    Gman.list.to_h.each do |group, domains|
-      next if WHITELIST.include?(group)
-      Parallel.each(domains, in_threads: 4) do |domain|
-        invalid.push(domain) unless importer.valid_domain?(domain, options)
-      end
-    end
-    assert_equal [], invalid.flatten.reject(&:empty?)
-  end
-end

data/test/test_gman_filter.rb

@@ -1,17 +0,0 @@
-HERE = File.dirname(__FILE__)
-require File.join(HERE, 'helper')
-
-class TestGmanFilter < Minitest::Test
-  txt_path = fixture_path 'obama.txt'
-  exec_path = bin_path 'gman_filter'
-
-  should 'remove non-gov/mil addresses' do
-    output, _status = Open3.capture2e('bundle', 'exec', exec_path, txt_path)
-    expected = %w(
-      mr.senator@obama.senate.gov
-      president@whitehouse.gov
-      commander.in.chief@us.army.mil
-    ).join("\n") + "\n"
-    assert_equal output, expected
-  end
-end

data/test/test_gman_identifier.rb

@@ -1,106 +0,0 @@
-require File.join(File.dirname(__FILE__), 'helper')
-
-class TestGmanIdentifier < Minitest::Test
-  should 'Parse the dotgov list' do
-    assert Gman.dotgov_list
-    assert_equal CSV::Table, Gman.dotgov_list.class
-    assert_equal CSV::Row, Gman.dotgov_list.first.class
-    assert Gman.dotgov_list.first['Domain Name']
-  end
-
-  context 'locality domains' do
-    should 'detect state domains' do
-      domain = Gman.new('state.ak.us')
-      assert domain.state?
-
-      refute domain.dotgov?
-      refute domain.city?
-      refute domain.federal?
-      refute domain.county?
-
-      assert_equal :state, domain.type
-      assert_equal 'AK', domain.state
-    end
-
-    should 'detect city domains' do
-      domain = Gman.new('ci.champaign.il.us')
-      assert domain.city?
-
-      refute domain.dotgov?
-      refute domain.state?
-      refute domain.federal?
-      refute domain.county?
-
-      assert_equal :city, domain.type
-      assert_equal 'IL', domain.state
-    end
-  end
-
-  context 'dotgovs' do
-    should 'detect federal dotgovs' do
-      domain = Gman.new 'whitehouse.gov'
-      assert domain.federal?
-      assert domain.dotgov?
-
-      refute domain.city?
-      refute domain.state?
-      refute domain.county?
-
-      assert_equal :federal, domain.type
-      assert_equal 'DC', domain.state
-      assert_equal 'Washington', domain.city
-      assert_equal 'Executive Office of the President', domain.agency
-    end
-
-    should 'detect state dotgovs' do
-      domain = Gman.new 'illinois.gov'
-      assert domain.state?
-      assert domain.dotgov?
-
-      refute domain.city?
-      refute domain.federal?
-      refute domain.county?
-
-      assert_equal :state, domain.type
-      assert_equal 'IL', domain.state
-      assert_equal 'Springfield', domain.city
-    end
-
-    should 'detect county dotgovs' do
-      domain = Gman.new 'ALLEGHENYCOUNTYPA.GOV'
-      assert domain.county?
-      assert domain.dotgov?
-
-      refute domain.city?
-      refute domain.federal?
-      refute domain.state?
-
-      assert_equal :county, domain.type
-      assert_equal 'PA', domain.state
-      assert_equal 'Pittsburgh', domain.city
-    end
-
-    should 'detect the list category' do
-      category = Gman.new('whitehouse.gov').send('list_category')
-      assert_equal 'US Federal', category
-    end
-  end
-
-  context 'non-dotgov domains' do
-    should "determine a domain's group" do
-      assert_equal 'usagovIN', Gman.new('cityofperu.org').send('list_category')
-      assert_equal :unknown, Gman.new('cityofperu.org').type
-
-      assert_equal 'Canada municipal', Gman.new('acme.ca').send('list_category')
-      assert_equal :"Canada municipal", Gman.new('acme.ca').type
-
-      assert_equal 'Canada federal', Gman.new('canada.ca').send('list_category')
-      assert_equal :"Canada federal", Gman.new('canada.ca').type
-    end
-
-    should 'detect the state' do
-      assert_equal 'OR', Gman.new('ashland.or.us').state
-      refute Gman.new('canada.ca').state
-    end
-  end
-end

data/test/test_gman_importer.rb

@@ -1,244 +0,0 @@
-require File.join(File.dirname(__FILE__), 'helper')
-
-class TestGManImporter < Minitest::Test
-  def setup
-    @importer = Gman::Importer.new 'test' => ['example.com']
-    @stdout = StringIO.new
-    @importer.instance_variable_set '@logger', Logger.new(@stdout)
-
-    @original_domain_list = File.read(stubbed_list_path)
-  end
-
-  def teardown
-    File.write stubbed_list_path, @original_domain_list
-  end
-
-  should 'init the domain list' do
-    assert_equal Gman::DomainList, @importer.domain_list.class
-    assert_equal 1, @importer.domain_list.count
-    assert_equal 'example.com', @importer.domain_list.domains.first
-  end
-
-  should 'init the logger' do
-    assert_equal Logger, @importer.logger.class
-  end
-
-  should 'return the current domain list' do
-    assert_equal Gman::DomainList, @importer.current.class
-  end
-
-  should 'return the resolver' do
-    assert_equal Resolv::DNS, @importer.resolver.class
-  end
-
-  context 'domain rejection' do
-    should 'return false for a rejected domain' do
-      refute @importer.reject 'example.com', 'reasons'
-    end
-
-    should 'return the reason when asked' do
-      with_env 'RECONCILING', 'true' do
-        assert_equal 'reasons', @importer.reject('example.com', 'reasons')
-      end
-    end
-  end
-
-  context 'manipulating the domain list' do
-    should 'normalize domains within the domain list' do
-      importer = Gman::Importer.new 'test' => ['www.EXAMPLE.com/']
-      importer.send :normalize_domains!
-      assert_equal 'example.com', importer.domain_list.domains.first
-    end
-
-    should 'remove invalid domains from the domain list' do
-      importer = Gman::Importer.new 'test' => ['foo.github.io', 'example.com']
-      importer.instance_variable_set '@logger', Logger.new(@stdout)
-
-      assert_equal 2, importer.domain_list.count
-      importer.send :ensure_validity!
-      assert_equal 1, importer.domain_list.count
-    end
-
-    context 'writing the domain list' do
-      should 'add domains to the current domain list' do
-        domains = { 'test' => ['example.com'], 'test2' => ['github.com'] }
-        importer = Gman::Importer.new domains
-        importer.instance_variable_set '@current', stubbed_list
-        importer.send :add_to_current
-        expected = "// test\nexample.com\ngov\n\n// test2\ngithub.com"
-        assert_equal expected, File.open(stubbed_list_path).read
-      end
-
-      should 'import' do
-        domains = {
-          'test'  => ['www.example.com', 'goo.github.io'],
-          'test2' => ['github.com', 'www.github.com', 'whitehouse.gov']
-        }
-
-        importer = Gman::Importer.new domains
-        importer.instance_variable_set '@current', stubbed_list
-        importer.instance_variable_set '@logger', Logger.new(@stdout)
-        importer.import(skip_resolve: true)
-
-        expected = "// test\nexample.com\ngov\n\n// test2\ngithub.com"
-        assert_equal expected, File.open(stubbed_list_path).read
-      end
-    end
-  end
-
-  context 'domain validation' do
-    should 'allow valid domains' do
-      assert @importer.send :ensure_valid, 'whitehouse.gov'
-    end
-
-    should 'reject empty domains' do
-      refute @importer.send :ensure_valid, ''
-    end
-
-    should 'reject blacklisted domains' do
-      refute @importer.send :ensure_valid, 'egovlink.com'
-    end
-
-    should 'reject invalid domains' do
-      refute @importer.send :ensure_valid, 'foo.invalid'
-    end
-
-    should 'reject academic domains' do
-      refute @importer.send :ensure_valid, 'harvard.edu'
-    end
-
-    should "reject regex'd domains" do
-      refute @importer.send :ensure_valid, 'foo.github.io'
-    end
-  end
-
-  context 'duplicate domains' do
-    should 'know a unique domain is not a dupe' do
-      refute @importer.send :dupe?, 'gman.com'
-    end
-
-    should "know when a domain's a dupe" do
-      assert @importer.send :dupe?, 'gov'
-    end
-
-    should "know when a domain's a subdomain of an existing domain" do
-      assert @importer.send :dupe?, 'whitehouse.gov'
-    end
-
-    should 'allow unique domains' do
-      assert @importer.send :ensure_not_dupe, 'gman.com'
-    end
-
-    should 'reject duplicate domains' do
-      refute @importer.send :ensure_not_dupe, 'gov'
-    end
-
-    should 'reject subdomains' do
-      refute @importer.send :ensure_not_dupe, 'whitehouse.gov'
-    end
-  end
-
-  context 'domain resolution' do
-    should 'know if a domain resolves' do
-      assert @importer.domain_resolves?('github.com')
-      assert @importer.send :ensure_resolves, 'github.com'
-    end
-
-    should "know if a domain doesn't resolve" do
-      refute @importer.domain_resolves?('foo.invalid')
-      refute @importer.send :ensure_resolves, 'foo.invalid'
-    end
-
-    should 'know if a domain has an IP' do
-    end
-
-    should 'know if a domain returns a given record' do
-    end
-  end
-
-  context 'regex checks' do
-    should 'pass valid domains' do
-      assert @importer.send :ensure_regex, 'example.com'
-    end
-
-    should 'reject domains that begin with home.' do
-      refute @importer.send :ensure_regex, 'home.example.com'
-    end
-
-    should 'reject domains that begin with user.' do
-      refute @importer.send :ensure_regex, 'user.example.com'
-    end
-
-    should 'reject domains that begin with site.' do
-      refute @importer.send :ensure_regex, 'user.example.com'
-    end
-
-    should 'reject weebly domains' do
-      refute @importer.send :ensure_regex, 'foo.weebly.com'
-    end
-
-    should 'reject wordpress domains' do
-      refute @importer.send :ensure_regex, 'foo.wordpress.com'
-    end
-
-    should 'reject govoffice domains' do
-      refute @importer.send :ensure_regex, 'foo.govoffice.com'
-      refute @importer.send :ensure_regex, 'foo.govoffice1.com'
-    end
-
-    should 'reject homestead domains' do
-      refute @importer.send :ensure_regex, 'foo.homestead.com'
-    end
-
-    should 'reject wix domains' do
-      refute @importer.send :ensure_regex, 'foo.wix.com'
-    end
-
-    should 'reject blogspot domains' do
-      refute @importer.send :ensure_regex, 'foo.blogspot.com'
-    end
-
-    should 'reject tripod domains' do
-      refute @importer.send :ensure_regex, 'foo.tripod.com'
-    end
-
-    should 'reject squarespace domains' do
-      refute @importer.send :ensure_regex, 'foo.squarespace.com'
-    end
-
-    should 'reject github.io domains' do
-      refute @importer.send :ensure_regex, 'foo.github.io'
-    end
-
-    should 'reject locality domains' do
-      refute @importer.send :ensure_regex, 'ci.champaign.il.us'
-    end
-  end
-
-  context 'normalizing domains' do
-    should 'normalize URLs to domains' do
-      expected = 'example.com'
-      assert_equal expected, @importer.normalize_domain('http://example.com')
-    end
-
-    should 'strip WWW' do
-      assert_equal 'example.com', @importer.normalize_domain('www.example.com')
-    end
-
-    should 'remove trailing slashes' do
-      assert_equal 'example.com', @importer.normalize_domain('example.com/')
-    end
-
-    should 'remove paths' do
-      assert_equal 'example.com', @importer.normalize_domain('example.com/foo')
-    end
-
-    should 'remove paths with trailing slashes' do
-      assert_equal 'example.com', @importer.normalize_domain('example.com/foo/')
-    end
-
-    should 'downcase' do
-      assert_equal 'example.com', @importer.normalize_domain('EXAMPLE.com')
-    end
-  end
-end