gman 7.0.0 → 7.0.5

data/CONTRIBUTING.md

@@ -1,22 +0,0 @@
-# Contributing to Gman
-
-## How to contribute
-
-1. Fork the project
-2. Create a descriptive branch
-3. Make your change
-4. Submit a pull request
-
-## Code
-
-Open an issue, or submit a pull request
-
-## Domains
-
-Domains live in `./config/domains.txt` as a list of TLDs and SLD+TLDs.
-
-Right now, the only valid government top level domains (TLDs), represent the US government and are `.gov`, and `.mil`.
-
-Secondary domains (e.g., `gov.uk`, or `mil.au`) detect non-US government entities.
-
-To add or remove a domain from the list of known government domains, simply edit the `domains.txt` file.

data/Rakefile

@@ -1,22 +0,0 @@
-require 'rubygems'
-require 'bundler'
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts 'Run `bundle install` to install missing gems'
-  exit e.status_code
-end
-require 'rake'
-
-require 'rake/testtask'
-Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
-  test.pattern = 'test/**/test_gman*.rb'
-  test.verbose = true
-end
-
-desc 'Open console with gman loaded'
-task :console do
-  exec 'irb -r ./lib/gman.rb'
-end

data/test/fixtures/domains.txt

@@ -1,2 +0,0 @@
-// test
-gov

data/test/helper.rb

@@ -1,48 +0,0 @@
-require 'rubygems'
-require 'bundler'
-require 'minitest/autorun'
-require 'parallel'
-require 'open3'
-
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts 'Run `bundle install` to install missing gems'
-  exit e.status_code
-end
-
-require 'shoulda'
-
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-$LOAD_PATH.unshift(File.dirname(__FILE__))
-require_relative '../lib/gman'
-require_relative '../lib/gman/domain_list'
-require_relative '../lib/gman/importer'
-
-def bin_path(cmd = 'gman')
-  File.expand_path "../bin/#{cmd}", File.dirname(__FILE__)
-end
-
-def test_bin(*args)
-  Open3.capture2e('bundle', 'exec', bin_path, *args)
-end
-
-def fixture_path(fixture)
-  File.expand_path "./fixtures/#{fixture}", File.dirname(__FILE__)
-end
-
-def with_env(key, value)
-  old_env = ENV[key]
-  ENV[key] = value
-  yield
-  ENV[key] = old_env
-end
-
-def stubbed_list_path
-  File.expand_path './fixtures/domains.txt', File.dirname(__FILE__)
-end
-
-def stubbed_list
-  Gman::DomainList.new(path: stubbed_list_path)
-end

data/test/test_gman.rb

@@ -1,56 +0,0 @@
-require File.join(File.dirname(__FILE__), 'helper')
-
-VALID = ['foo.gov',
-         'http://foo.mil',
-         'foo@bar.gc.ca',
-         'foo.gov.au',
-         'https://www.foo.gouv.fr',
-         'foo@ci.champaign.il.us',
-         'foo.bar.baz.gov.au',
-         'foo@bar.gov.uk',
-         'foo.gov',
-         'foo.fed.us',
-         'foo.state.il.us',
-         'state.il.us',
-         'foo@af.mil',
-         'foo.gov.in'
-        ].freeze
-
-INVALID = ['foo.bar.com',
-           'bar@foo.biz',
-           'http://www.foo.biz',
-           'foo.uk',
-           'gov',
-           'foo@k12.champaign.il.us',
-           'foo@kii.gov.by',
-           'foo',
-           '',
-           nil,
-           ' ',
-           'foo.city.il.us',
-           'foo.ci.il.us',
-           'foo.zx.us',
-           'foo@mail.gov.ua'
-          ].freeze
-
-class TestGman < Minitest::Test
-  VALID.each do |domain|
-    should "recognize #{domain} as a government domain" do
-      assert Gman.valid?(domain)
-    end
-  end
-
-  INVALID.each do |domain|
-    should "recognize #{domain} as a non-government domain" do
-      refute Gman.valid?(domain)
-    end
-  end
-
-  should 'not allow educational domains' do
-    assert_equal false, Gman.valid?('foo@gwu.edu')
-  end
-
-  should 'returns the path to domains.txt' do
-    assert_equal true, File.exist?(Gman.list_path)
-  end
-end

data/test/test_gman_bin.rb

@@ -1,75 +0,0 @@
-require_relative 'helper'
-
-class TestGmanBin < Minitest::Test
-  def setup
-    @output, @status = test_bin('whitehouse.gov')
-  end
-
-  should 'parse the domain' do
-    output, = test_bin('bar.gov')
-    assert_match(/Domain  : bar.gov/, output)
-
-    output, = test_bin('foo@bar.gov')
-    assert_match(/Domain  : bar.gov/, output)
-
-    output, = test_bin('http://bar.gov/foo')
-    assert_match(/Domain  : bar.gov/, output)
-  end
-
-  should 'err on invalid domains' do
-    output, status = test_bin('foo.invalid')
-    assert_equal 1, status.exitstatus
-    assert_match(/Invalid domain/, output)
-  end
-
-  should 'err on non-government domains' do
-    output, status = test_bin('github.com')
-    assert_equal 1, status.exitstatus
-    assert_match(/Not a government domain/, output)
-  end
-
-  should 'know the type' do
-    assert_match(/federal/, @output)
-    assert_equal 0, @status.exitstatus
-  end
-
-  should 'know the agency' do
-    assert_match(/Executive Office of the President/, @output)
-    assert_equal 0, @status.exitstatus
-  end
-
-  should 'know the country' do
-    assert_match(/United States/, @output)
-    assert_equal 0, @status.exitstatus
-  end
-
-  should 'know the city' do
-    assert_match(/Washington/, @output)
-    assert_equal 0, @status.exitstatus
-  end
-
-  should 'know the state' do
-    assert_match(/DC/, @output)
-    assert_equal 0, @status.exitstatus
-  end
-
-  should 'allow you to disable colorization' do
-    output, = test_bin('whitehouse.gov', '--no-color')
-    refute_match(/\e\[32m/, output)
-  end
-
-  should 'color by default' do
-    assert_match(/\e\[32m/, @output)
-  end
-
-  should 'show help text' do
-    output, = test_bin
-    assert_match(/Usage/i, output)
-
-    output, = test_bin('')
-    assert_match(/Usage/i, output)
-
-    output, = test_bin('--no-color')
-    assert_match(/Usage/i, output)
-  end
-end

data/test/test_gman_country_codes.rb

@@ -1,18 +0,0 @@
-require File.join(File.dirname(__FILE__), 'helper')
-
-class TestGmanCountryCodes < Minitest::Test
-  should "determine a domain's country" do
-    name = Gman.new('whitehouse.gov').country.name
-    assert_equal 'United States of America', name
-
-    name = Gman.new('foo.gov.uk').country.name
-    assert_equal 'United Kingdom of Great Britain and Northern Ireland', name
-
-    assert_equal 'United States of America', Gman.new('army.mil').country.name
-    assert_equal 'Canada', Gman.new('foo.gc.ca').country.name
-  end
-
-  should 'not err out on an unknown country code' do
-    assert_equal nil, Gman.new('foo.eu').country
-  end
-end

data/test/test_gman_domain_list.rb

@@ -1,112 +0,0 @@
-require File.join(File.dirname(__FILE__), 'helper')
-
-class TestGmanDomainList < Minitest::Test
-  INIT_TYPES = [:path, :contents, :data].freeze
-
-  def setup
-    @original_domain_list = File.read(stubbed_list_path)
-  end
-
-  def teardown
-    File.write stubbed_list_path, @original_domain_list
-  end
-
-  def domain_list(type)
-    case type
-    when :path
-      Gman::DomainList.new(path: Gman.list_path)
-    when :contents
-      contents = File.read(Gman.list_path)
-      Gman::DomainList.new(contents: contents)
-    when :data
-      data = Gman::DomainList.new(path: Gman.list_path).to_h
-      Gman::DomainList.new(data: data)
-    end
-  end
-
-  INIT_TYPES.each do |type|
-    context "when initalized with #{type}" do
-      should 'store the init vars' do
-        refute domain_list(type).public_send(type).nil?
-      end
-
-      should 'return the domain data' do
-        list = domain_list(type)
-        assert list.data.key? 'Canada federal'
-        assert list.data.any? { |_key, values| values.include? 'gov' }
-      end
-
-      should 'return the list contents' do
-        list = domain_list(type)
-        assert_match(/^gov$/, list.contents)
-      end
-
-      should 'return the list path' do
-        list = domain_list(type)
-        assert_equal list.path, Gman.list_path
-      end
-
-      should 'return the public suffix parsed list' do
-        list = domain_list(type)
-        assert list.public_suffix_list.class == PublicSuffix::List
-      end
-
-      should 'know if a domain is valid' do
-        list = domain_list(type)
-        assert list.valid? 'whitehouse.gov'
-      end
-
-      should 'know if a domain is invalid' do
-        list = domain_list(type)
-        refute list.valid? 'example.com'
-      end
-
-      should 'return the domain groups' do
-        list = domain_list(type)
-        assert list.groups.include?('Canada federal')
-      end
-
-      should 'return the domains' do
-        list = domain_list(type)
-        assert list.domains.include?('gov')
-      end
-
-      should 'return the domain count' do
-        list = domain_list(type)
-        assert list.count.is_a?(Integer)
-        assert list.count > 100
-      end
-
-      should 'alphabetize the list' do
-        list = domain_list(type)
-        list.data['Canada municipal'].shuffle!
-        assert list.data['Canada municipal'].first != '100milehouse.com'
-        list.alphabetize
-        assert list.data['Canada municipal'].first == '100milehouse.com'
-      end
-
-      should 'write the list' do
-        list = domain_list(type)
-        list.instance_variable_set('@path', stubbed_list_path)
-        list.data = { 'foo' => ['bar.gov', 'baz.net'] }
-        list.write
-        contents = File.read(stubbed_list_path)
-        assert_match %r{^// foo$}, contents
-        expected = "// foo\nbar.gov\nbaz.net"
-        assert contents.include?(expected)
-      end
-
-      should 'output the list in public_suffix format' do
-        list = domain_list(type)
-        string = list.to_s
-        assert_match %r{^// Canada federal$}, string
-        assert string.include? "// Canada federal\ncanada\.ca\n"
-      end
-
-      should "find a domain's parent" do
-        list = domain_list(type)
-        assert_equal 'gov.uk', list.parent_domain('foo.gov.uk')
-      end
-    end
-  end
-end

data/test/test_gman_domains.rb

@@ -1,32 +0,0 @@
-require File.join(File.dirname(__FILE__), 'helper')
-
-class TestGmanDomains < Minitest::Test
-  WHITELIST = ['non-us gov', 'non-us mil', 'US Federal'].freeze
-
-  def resolve_domains?
-    ENV['GMAN_RESOLVE_DOMAINS'] == 'true'
-  end
-
-  should 'only contains valid domains' do
-    importer = Gman::Importer.new({})
-    if resolve_domains?
-      importer.logger.info <<-MSG
-        Validating that all domains resolve. This may take a while...
-      MSG
-    else
-      importer.logger.info 'Skipping domain resolution.' \
-        'Run `GMAN_RESOLVE_DOMAINS=true rake test` '\
-        'to validate that domains resolve.'
-    end
-
-    invalid = []
-    options = { skip_dupe: true, skip_resolve: !resolve_domains? }
-    Gman.list.to_h.each do |group, domains|
-      next if WHITELIST.include?(group)
-      Parallel.each(domains, in_threads: 4) do |domain|
-        invalid.push(domain) unless importer.valid_domain?(domain, options)
-      end
-    end
-    assert_equal [], invalid.flatten.reject(&:empty?)
-  end
-end

data/test/test_gman_filter.rb

@@ -1,17 +0,0 @@
-HERE = File.dirname(__FILE__)
-require File.join(HERE, 'helper')
-
-class TestGmanFilter < Minitest::Test
-  txt_path = fixture_path 'obama.txt'
-  exec_path = bin_path 'gman_filter'
-
-  should 'remove non-gov/mil addresses' do
-    output, _status = Open3.capture2e('bundle', 'exec', exec_path, txt_path)
-    expected = %w(
-      mr.senator@obama.senate.gov
-      president@whitehouse.gov
-      commander.in.chief@us.army.mil
-    ).join("\n") + "\n"
-    assert_equal output, expected
-  end
-end

data/test/test_gman_identifier.rb

@@ -1,106 +0,0 @@
-require File.join(File.dirname(__FILE__), 'helper')
-
-class TestGmanIdentifier < Minitest::Test
-  should 'Parse the dotgov list' do
-    assert Gman.dotgov_list
-    assert_equal CSV::Table, Gman.dotgov_list.class
-    assert_equal CSV::Row, Gman.dotgov_list.first.class
-    assert Gman.dotgov_list.first['Domain Name']
-  end
-
-  context 'locality domains' do
-    should 'detect state domains' do
-      domain = Gman.new('state.ak.us')
-      assert domain.state?
-
-      refute domain.dotgov?
-      refute domain.city?
-      refute domain.federal?
-      refute domain.county?
-
-      assert_equal :state, domain.type
-      assert_equal 'AK', domain.state
-    end
-
-    should 'detect city domains' do
-      domain = Gman.new('ci.champaign.il.us')
-      assert domain.city?
-
-      refute domain.dotgov?
-      refute domain.state?
-      refute domain.federal?
-      refute domain.county?
-
-      assert_equal :city, domain.type
-      assert_equal 'IL', domain.state
-    end
-  end
-
-  context 'dotgovs' do
-    should 'detect federal dotgovs' do
-      domain = Gman.new 'whitehouse.gov'
-      assert domain.federal?
-      assert domain.dotgov?
-
-      refute domain.city?
-      refute domain.state?
-      refute domain.county?
-
-      assert_equal :federal, domain.type
-      assert_equal 'DC', domain.state
-      assert_equal 'Washington', domain.city
-      assert_equal 'Executive Office of the President', domain.agency
-    end
-
-    should 'detect state dotgovs' do
-      domain = Gman.new 'illinois.gov'
-      assert domain.state?
-      assert domain.dotgov?
-
-      refute domain.city?
-      refute domain.federal?
-      refute domain.county?
-
-      assert_equal :state, domain.type
-      assert_equal 'IL', domain.state
-      assert_equal 'Springfield', domain.city
-    end
-
-    should 'detect county dotgovs' do
-      domain = Gman.new 'ALLEGHENYCOUNTYPA.GOV'
-      assert domain.county?
-      assert domain.dotgov?
-
-      refute domain.city?
-      refute domain.federal?
-      refute domain.state?
-
-      assert_equal :county, domain.type
-      assert_equal 'PA', domain.state
-      assert_equal 'Pittsburgh', domain.city
-    end
-
-    should 'detect the list category' do
-      category = Gman.new('whitehouse.gov').send('list_category')
-      assert_equal 'US Federal', category
-    end
-  end
-
-  context 'non-dotgov domains' do
-    should "determine a domain's group" do
-      assert_equal 'usagovIN', Gman.new('cityofperu.org').send('list_category')
-      assert_equal :unknown, Gman.new('cityofperu.org').type
-
-      assert_equal 'Canada municipal', Gman.new('acme.ca').send('list_category')
-      assert_equal :"Canada municipal", Gman.new('acme.ca').type
-
-      assert_equal 'Canada federal', Gman.new('canada.ca').send('list_category')
-      assert_equal :"Canada federal", Gman.new('canada.ca').type
-    end
-
-    should 'detect the state' do
-      assert_equal 'OR', Gman.new('ashland.or.us').state
-      refute Gman.new('canada.ca').state
-    end
-  end
-end