gman 6.0.1 → 7.0.4

data/CONTRIBUTING.md

@@ -1,22 +0,0 @@
-# Contributing to Gman
-
-## How to contribute
-
-1. Fork the project
-2. Create a descriptive branch
-3. Make your change
-4. Submit a pull request
-
-## Code
-
-Open an issue, or submit a pull request
-
-## Domains
-
-Domains live in `./config/domains.txt` as a list of TLDs and SLD+TLDs.
-
-Right now, the only valid government top level domains (TLDs), represent the US government and are `.gov`, and `.mil`.
-
-Secondary domains (e.g., `gov.uk`, or `mil.au`) detect non-US government entities.
-
-To add or remove a domain from the list of known government domains, simply edit the `domains.txt` file.

data/Rakefile

@@ -1,22 +0,0 @@
-require 'rubygems'
-require 'bundler'
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts 'Run `bundle install` to install missing gems'
-  exit e.status_code
-end
-require 'rake'
-
-require 'rake/testtask'
-Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
-  test.pattern = 'test/**/test_gman*.rb'
-  test.verbose = true
-end
-
-desc 'Open console with gman loaded'
-task :console do
-  exec 'irb -r ./lib/gman.rb'
-end

data/test/fixtures/domains.txt

@@ -1,2 +0,0 @@
-// test
-gov

data/test/helper.rb

@@ -1,40 +0,0 @@
-require 'rubygems'
-require 'bundler'
-require 'minitest/autorun'
-require 'parallel'
-require 'open3'
-
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts 'Run `bundle install` to install missing gems'
-  exit e.status_code
-end
-
-require 'shoulda'
-
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-$LOAD_PATH.unshift(File.dirname(__FILE__))
-require_relative '../lib/gman'
-require_relative '../lib/gman/domain_list'
-require_relative '../lib/gman/importer'
-
-def bin_path(cmd = 'gman')
-  File.expand_path "../bin/#{cmd}", File.dirname(__FILE__)
-end
-
-def test_bin(*args)
-  Open3.capture2e('bundle', 'exec', bin_path, *args)
-end
-
-def fixture_path(fixture)
-  File.expand_path "./fixtures/#{fixture}", File.dirname(__FILE__)
-end
-
-def with_env(key, value)
-  old_env = ENV[key]
-  ENV[key] = value
-  yield
-  ENV[key] = old_env
-end

data/test/test_gman.rb

@@ -1,62 +0,0 @@
-require File.join(File.dirname(__FILE__), 'helper')
-
-VALID = ['foo.gov',
-         'http://foo.mil',
-         'foo@bar.gc.ca',
-         'foo.gov.au',
-         'https://www.foo.gouv.fr',
-         'foo@ci.champaign.il.us',
-         'foo.bar.baz.gov.au',
-         'foo@bar.gov.uk',
-         'foo.gov',
-         'foo.fed.us',
-         'foo.state.il.us',
-         'state.il.us',
-         'foo@af.mil',
-         'foo.gov.in'
-        ].freeze
-
-INVALID = ['foo.bar.com',
-           'bar@foo.biz',
-           'http://www.foo.biz',
-           'foo.uk',
-           'gov',
-           'foo@k12.champaign.il.us',
-           'foo@kii.gov.by',
-           'foo',
-           '',
-           nil,
-           ' ',
-           'foo.city.il.us',
-           'foo.ci.il.us',
-           'foo.zx.us',
-           'foo@mail.gov.ua'
-          ].freeze
-
-class TestGman < Minitest::Test
-  VALID.each do |domain|
-    should "recognize #{domain} as a government domain" do
-      assert Gman.valid?(domain)
-    end
-  end
-
-  INVALID.each do |domain|
-    should "recognize #{domain} as a non-government domain" do
-      refute Gman.valid?(domain)
-    end
-  end
-
-  should 'not allow educational domains' do
-    assert_equal false, Gman.valid?('foo@gwu.edu')
-  end
-
-  should 'returns the path to domains.txt' do
-    assert_equal true, File.exist?(Gman.list_path)
-  end
-
-  should 'stub domains when asked' do
-    with_env 'GMAN_STUB_DOMAINS', 'true' do
-      assert_equal fixture_path('domains.txt'), Gman.list_path
-    end
-  end
-end

data/test/test_gman_bin.rb

@@ -1,75 +0,0 @@
-require_relative 'helper'
-
-class TestGmanBin < Minitest::Test
-  def setup
-    @output, @status = test_bin('whitehouse.gov')
-  end
-
-  should 'parse the domain' do
-    output, = test_bin('bar.gov')
-    assert_match(/Domain  : bar.gov/, output)
-
-    output, = test_bin('foo@bar.gov')
-    assert_match(/Domain  : bar.gov/, output)
-
-    output, = test_bin('http://bar.gov/foo')
-    assert_match(/Domain  : bar.gov/, output)
-  end
-
-  should 'err on invalid domains' do
-    output, status = test_bin('foo.invalid')
-    assert_equal 1, status.exitstatus
-    assert_match(/Invalid domain/, output)
-  end
-
-  should 'err on non-government domains' do
-    output, status = test_bin('github.com')
-    assert_equal 1, status.exitstatus
-    assert_match(/Not a government domain/, output)
-  end
-
-  should 'know the type' do
-    assert_match(/federal/, @output)
-    assert_equal 0, @status.exitstatus
-  end
-
-  should 'know the agency' do
-    assert_match(/Executive Office of the President/, @output)
-    assert_equal 0, @status.exitstatus
-  end
-
-  should 'know the country' do
-    assert_match(/United States/, @output)
-    assert_equal 0, @status.exitstatus
-  end
-
-  should 'know the city' do
-    assert_match(/Washington/, @output)
-    assert_equal 0, @status.exitstatus
-  end
-
-  should 'know the state' do
-    assert_match(/DC/, @output)
-    assert_equal 0, @status.exitstatus
-  end
-
-  should 'allow you to disable colorization' do
-    output, = test_bin('whitehouse.gov', '--no-color')
-    refute_match(/\e\[32m/, output)
-  end
-
-  should 'color by default' do
-    assert_match(/\e\[32m/, @output)
-  end
-
-  should 'show help text' do
-    output, = test_bin
-    assert_match(/Usage/i, output)
-
-    output, = test_bin('')
-    assert_match(/Usage/i, output)
-
-    output, = test_bin('--no-color')
-    assert_match(/Usage/i, output)
-  end
-end

data/test/test_gman_country_codes.rb

@@ -1,18 +0,0 @@
-require File.join(File.dirname(__FILE__), 'helper')
-
-class TestGmanCountryCodes < Minitest::Test
-  should "determine a domain's country" do
-    name = Gman.new('whitehouse.gov').country.name
-    assert_equal 'United States of America', name
-
-    name = Gman.new('foo.gov.uk').country.name
-    assert_equal 'United Kingdom of Great Britain and Northern Ireland', name
-
-    assert_equal 'United States of America', Gman.new('army.mil').country.name
-    assert_equal 'Canada', Gman.new('foo.gc.ca').country.name
-  end
-
-  should 'not err out on an unknown country code' do
-    assert_equal nil, Gman.new('foo.eu').country
-  end
-end

data/test/test_gman_domains.rb

@@ -1,33 +0,0 @@
-require File.join(File.dirname(__FILE__), 'helper')
-
-class TestGmanDomains < Minitest::Test
-  WHITELIST = ['non-us gov', 'non-us mil', 'US Federal'].freeze
-
-  def resolve_domains?
-    ENV['GMAN_RESOLVE_DOMAINS'] == 'true'
-  end
-
-  should 'only contains valid domains' do
-    importer = Gman::Importer.new({})
-    if resolve_domains?
-      importer.logger.info <<-MSG
-        Validating that all domains resolve. This may take a while...
-      MSG
-    else
-      importer.logger.info 'Skipping domain resolution.' \
-        'Run `GMAN_RESOLVE_DOMAINS=true rake test` '\
-        'to validate that domains resolve.'
-    end
-
-    invalid = []
-    list = Gman::DomainList.current.list
-    Parallel.each(list, in_threads: 2) do |group, domains|
-      next if WHITELIST.include?(group)
-      invalid.push domains.reject { |domain|
-        options = { skip_dupe: true, skip_resolve: !resolve_domains? }
-        importer.valid_domain?(domain, options)
-      }
-    end
-    assert_equal [], invalid.flatten.reject(&:empty?)
-  end
-end

data/test/test_gman_filter.rb

@@ -1,17 +0,0 @@
-HERE = File.dirname(__FILE__)
-require File.join(HERE, 'helper')
-
-class TestGmanFilter < Minitest::Test
-  txt_path = fixture_path 'obama.txt'
-  exec_path = bin_path 'gman_filter'
-
-  should 'remove non-gov/mil addresses' do
-    output, _status = Open3.capture2e('bundle', 'exec', exec_path, txt_path)
-    expected = %w(
-      mr.senator@obama.senate.gov
-      president@whitehouse.gov
-      commander.in.chief@us.army.mil
-    ).join("\n") + "\n"
-    assert_equal output, expected
-  end
-end

data/test/test_gman_identifier.rb

@@ -1,106 +0,0 @@
-require File.join(File.dirname(__FILE__), 'helper')
-
-class TestGmanIdentifier < Minitest::Test
-  should 'Parse the dotgov list' do
-    assert Gman.dotgov_list
-    assert_equal CSV::Table, Gman.dotgov_list.class
-    assert_equal CSV::Row, Gman.dotgov_list.first.class
-    assert Gman.dotgov_list.first['Domain Name']
-  end
-
-  context 'locality domains' do
-    should 'detect state domains' do
-      domain = Gman.new('state.ak.us')
-      assert domain.state?
-
-      refute domain.dotgov?
-      refute domain.city?
-      refute domain.federal?
-      refute domain.county?
-
-      assert_equal :state, domain.type
-      assert_equal 'AK', domain.state
-    end
-
-    should 'detect city domains' do
-      domain = Gman.new('ci.champaign.il.us')
-      assert domain.city?
-
-      refute domain.dotgov?
-      refute domain.state?
-      refute domain.federal?
-      refute domain.county?
-
-      assert_equal :city, domain.type
-      assert_equal 'IL', domain.state
-    end
-  end
-
-  context 'dotgovs' do
-    should 'detect federal dotgovs' do
-      domain = Gman.new 'whitehouse.gov'
-      assert domain.federal?
-      assert domain.dotgov?
-
-      refute domain.city?
-      refute domain.state?
-      refute domain.county?
-
-      assert_equal :federal, domain.type
-      assert_equal 'DC', domain.state
-      assert_equal 'Washington', domain.city
-      assert_equal 'Executive Office of the President', domain.agency
-    end
-
-    should 'detect state dotgovs' do
-      domain = Gman.new 'illinois.gov'
-      assert domain.state?
-      assert domain.dotgov?
-
-      refute domain.city?
-      refute domain.federal?
-      refute domain.county?
-
-      assert_equal :state, domain.type
-      assert_equal 'IL', domain.state
-      assert_equal 'Springfield', domain.city
-    end
-
-    should 'detect county dotgovs' do
-      domain = Gman.new 'ALLEGHENYCOUNTYPA.GOV'
-      assert domain.county?
-      assert domain.dotgov?
-
-      refute domain.city?
-      refute domain.federal?
-      refute domain.state?
-
-      assert_equal :county, domain.type
-      assert_equal 'PA', domain.state
-      assert_equal 'Pittsburgh', domain.city
-    end
-
-    should 'detect the list category' do
-      category = Gman.new('whitehouse.gov').send('list_category')
-      assert_equal 'US Federal', category
-    end
-  end
-
-  context 'non-dotgov domains' do
-    should "determine a domain's group" do
-      assert_equal 'usagovIN', Gman.new('cityofperu.org').send('list_category')
-      assert_equal :unknown, Gman.new('cityofperu.org').type
-
-      assert_equal 'Canada municipal', Gman.new('acme.ca').send('list_category')
-      assert_equal :"Canada municipal", Gman.new('acme.ca').type
-
-      assert_equal 'Canada federal', Gman.new('canada.ca').send('list_category')
-      assert_equal :"Canada federal", Gman.new('canada.ca').type
-    end
-
-    should 'detect the state' do
-      assert_equal 'OR', Gman.new('ashland.or.us').state
-      refute Gman.new('canada.ca').state
-    end
-  end
-end

data/test/test_gman_importer.rb

@@ -1,250 +0,0 @@
-require File.join(File.dirname(__FILE__), 'helper')
-
-class TestGManImporter < Minitest::Test
-  def setup
-    @importer = Gman::Importer.new 'test' => ['example.com']
-    @stdout = StringIO.new
-    @importer.instance_variable_set '@logger', Logger.new(@stdout)
-
-    with_env 'GMAN_STUB_DOMAINS', 'true' do
-      @original_domain_list = File.open(Gman.list_path).read
-    end
-  end
-
-  def teardown
-    with_env 'GMAN_STUB_DOMAINS', 'true' do
-      File.write Gman.list_path, @original_domain_list
-    end
-  end
-
-  should 'init the domain list' do
-    assert_equal Gman::DomainList, @importer.domains.class
-    assert_equal 1, @importer.domains.domains.count
-    assert_equal 'example.com', @importer.domains.domains.first
-  end
-
-  should 'init the logger' do
-    assert_equal Logger, @importer.logger.class
-  end
-
-  should 'return the current domain list' do
-    assert_equal Gman::DomainList, @importer.current.class
-  end
-
-  should 'return the resolver' do
-    assert_equal Resolv::DNS, @importer.resolver.class
-  end
-
-  context 'domain rejection' do
-    should 'return false for a rejected domain' do
-      refute @importer.reject 'example.com', 'reasons'
-    end
-
-    should 'return the reason when asked' do
-      with_env 'RECONCILING', 'true' do
-        assert_equal 'reasons', @importer.reject('example.com', 'reasons')
-      end
-    end
-  end
-
-  context 'manipulating the domain list' do
-    should 'normalize domains within the domain list' do
-      importer = Gman::Importer.new 'test' => ['www.EXAMPLE.com/']
-      importer.send :normalize_domains!
-      assert_equal 'example.com', importer.domains.domains.first
-    end
-
-    should 'remove invalid domains from the domain list' do
-      importer = Gman::Importer.new 'test' => ['foo.github.io', 'example.com']
-      importer.instance_variable_set '@logger', Logger.new(@stdout)
-
-      assert_equal 2, importer.domains.domains.count
-      importer.send :ensure_validity!
-      assert_equal 1, importer.domains.domains.count
-    end
-
-    context 'writing the domain list' do
-      should 'add domains to the current domain list' do
-        with_env 'GMAN_STUB_DOMAINS', 'true' do
-          domains = { 'test' => ['example.com'], 'test2' => ['github.com'] }
-          importer = Gman::Importer.new domains
-          importer.send :add_to_current
-          expected = "// test\nexample.com\ngov\n\n// test2\ngithub.com"
-          assert_equal expected, File.open(Gman.list_path).read
-        end
-      end
-
-      should 'import' do
-        with_env 'GMAN_STUB_DOMAINS', 'true' do
-          domains = {
-            'test'  => ['www.example.com', 'goo.github.io'],
-            'test2' => ['github.com', 'www.github.com', 'whitehouse.gov']
-          }
-
-          importer = Gman::Importer.new domains
-          importer.instance_variable_set '@logger', Logger.new(@stdout)
-          importer.import(skip_resolve: true)
-
-          expected = "// test\nexample.com\ngov\n\n// test2\ngithub.com"
-          assert_equal expected, File.open(Gman.list_path).read
-        end
-      end
-    end
-  end
-
-  context 'domain validation' do
-    should 'allow valid domains' do
-      assert @importer.send :ensure_valid, 'whitehouse.gov'
-    end
-
-    should 'reject empty domains' do
-      refute @importer.send :ensure_valid, ''
-    end
-
-    should 'reject blacklisted domains' do
-      refute @importer.send :ensure_valid, 'egovlink.com'
-    end
-
-    should 'reject invalid domains' do
-      refute @importer.send :ensure_valid, 'foo.invalid'
-    end
-
-    should 'reject academic domains' do
-      refute @importer.send :ensure_valid, 'harvard.edu'
-    end
-
-    should "reject regex'd domains" do
-      refute @importer.send :ensure_valid, 'foo.github.io'
-    end
-  end
-
-  context 'duplicate domains' do
-    should 'know a unique domain is not a dupe' do
-      refute @importer.send :dupe?, 'gman.com'
-    end
-
-    should "know when a domain's a dupe" do
-      assert @importer.send :dupe?, 'gov'
-    end
-
-    should "know when a domain's a subdomain of an existing domain" do
-      assert @importer.send :dupe?, 'whitehouse.gov'
-    end
-
-    should 'allow unique domains' do
-      assert @importer.send :ensure_not_dupe, 'gman.com'
-    end
-
-    should 'reject duplicate domains' do
-      refute @importer.send :ensure_not_dupe, 'gov'
-    end
-
-    should 'reject subdomains' do
-      refute @importer.send :ensure_not_dupe, 'whitehouse.gov'
-    end
-  end
-
-  context 'domain resolution' do
-    should 'know if a domain resolves' do
-      assert @importer.domain_resolves?('github.com')
-      assert @importer.send :ensure_resolves, 'github.com'
-    end
-
-    should "know if a domain doesn't resolve" do
-      refute @importer.domain_resolves?('foo.invalid')
-      refute @importer.send :ensure_resolves, 'foo.invalid'
-    end
-
-    should 'know if a domain has an IP' do
-    end
-
-    should 'know if a domain returns a given record' do
-    end
-  end
-
-  context 'regex checks' do
-    should 'pass valid domains' do
-      assert @importer.send :ensure_regex, 'example.com'
-    end
-
-    should 'reject domains that begin with home.' do
-      refute @importer.send :ensure_regex, 'home.example.com'
-    end
-
-    should 'reject domains that begin with user.' do
-      refute @importer.send :ensure_regex, 'user.example.com'
-    end
-
-    should 'reject domains that begin with site.' do
-      refute @importer.send :ensure_regex, 'user.example.com'
-    end
-
-    should 'reject weebly domains' do
-      refute @importer.send :ensure_regex, 'foo.weebly.com'
-    end
-
-    should 'reject wordpress domains' do
-      refute @importer.send :ensure_regex, 'foo.wordpress.com'
-    end
-
-    should 'reject govoffice domains' do
-      refute @importer.send :ensure_regex, 'foo.govoffice.com'
-      refute @importer.send :ensure_regex, 'foo.govoffice1.com'
-    end
-
-    should 'reject homestead domains' do
-      refute @importer.send :ensure_regex, 'foo.homestead.com'
-    end
-
-    should 'reject wix domains' do
-      refute @importer.send :ensure_regex, 'foo.wix.com'
-    end
-
-    should 'reject blogspot domains' do
-      refute @importer.send :ensure_regex, 'foo.blogspot.com'
-    end
-
-    should 'reject tripod domains' do
-      refute @importer.send :ensure_regex, 'foo.tripod.com'
-    end
-
-    should 'reject squarespace domains' do
-      refute @importer.send :ensure_regex, 'foo.squarespace.com'
-    end
-
-    should 'reject github.io domains' do
-      refute @importer.send :ensure_regex, 'foo.github.io'
-    end
-
-    should 'reject locality domains' do
-      refute @importer.send :ensure_regex, 'ci.champaign.il.us'
-    end
-  end
-
-  context 'normalizing domains' do
-    should 'normalize URLs to domains' do
-      expected = 'example.com'
-      assert_equal expected, @importer.normalize_domain('http://example.com')
-    end
-
-    should 'strip WWW' do
-      assert_equal 'example.com', @importer.normalize_domain('www.example.com')
-    end
-
-    should 'remove trailing slashes' do
-      assert_equal 'example.com', @importer.normalize_domain('example.com/')
-    end
-
-    should 'remove paths' do
-      assert_equal 'example.com', @importer.normalize_domain('example.com/foo')
-    end
-
-    should 'remove paths with trailing slashes' do
-      assert_equal 'example.com', @importer.normalize_domain('example.com/foo/')
-    end
-
-    should 'downcase' do
-      assert_equal 'example.com', @importer.normalize_domain('EXAMPLE.com')
-    end
-  end
-end