gman 7.0.6 → 8.1.0

data/script/console

@@ -1,3 +0,0 @@
-#! /bin/sh
-
-bundle exec pry -r ./lib/gman.rb

data/script/dedupe

@@ -1,22 +0,0 @@
-#! /usr/bin/env ruby
-# frozen_string_literal: true
-
-require 'yaml'
-require 'open-uri'
-require './lib/gman'
-require './lib/gman/importer'
-
-current = Gman::DomainList.current
-
-puts 'Checking for duplicate domains in the domain list...'
-puts "Current list contains #{current.count} domains..."
-
-dupe = current.count - current.domains.uniq.count
-puts "Found #{dupe} duplicate domains"
-exit 0 if dupe.zero?
-
-dupes = current.domains.select { |domain| current.domains.count(domain) > 1 }
-
-puts 'Duplicate domains:'
-puts dupes
-exit 1

data/script/profile

@@ -1,21 +0,0 @@
-#! /usr/bin/env ruby
-# frozen_string_literal: true
-
-require 'ruby-prof'
-require './lib/gman'
-
-# Pick N random domains directly,
-# without pre-loading the Gman list for an accurate benchmark
-count = (ARGV[0] || 100).to_i
-domains = File.readlines('./config/domains.txt')
-domains = domains.grep(/^[a-z0-9]/i)
-domains = domains.sample(count)
-
-RubyProf.start
-domains.each do |domain|
-  Gman.new("foo@bar.#{domain}").valid?
-end
-
-result = RubyProf.stop
-printer = RubyProf::FlatPrinter.new(result)
-printer.print($stdout)

data/script/prune

@@ -1,22 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-# Given an array of domains, removes them from the list
-# Example usage: script/prune foo.invalid, bar.invalid, foo.bar.invalid
-
-require_relative '../lib/gman'
-require_relative '../lib/gman/domain_list'
-
-domains = ARGV
-domains = domains.clone.map { |d| d.delete ',' }
-
-list = File.read('./config/domains.txt')
-puts "Starting list: #{Gman::DomainList.current.count} domains"
-
-domains.each do |domain|
-  list.gsub!(/^#{Regexp.escape(domain)}$\n/, '')
-end
-
-File.write './config/domains.txt', list
-
-puts "Ending list: #{Gman::DomainList.current.count} domains"

data/script/reconcile-us

@@ -1,72 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-#
-# Reconciles the USA.gov-maintained list of US domains with domains.txt
-# to show domains listed in the USA.gov-maintained list that we reject and why
-#
-# Usage: script/reconcile-us
-
-require './lib/gman/importer'
-require 'yaml'
-
-ENV['RECONCILING'] = 'true'
-blacklist = ['usagovQUASI']
-source = 'https://raw.githubusercontent.com/GSA/govt-urls/master/government-urls-hierarchical-list.txt'
-
-data = URI.open(source).read
-data = data.split('_' * 74)
-data = data.last.strip
-data = data.split(/\r?\n/).reject(&:empty?)
-
-domains = {}
-group = ''
-data.each do |row|
-  if /^\w/.match?(row)
-    group = row
-    domains[group] = []
-  else
-    domains[group].push row.sub(".\t", '').strip
-  end
-end
-
-domains.reject! { |g, _domain| blacklist.include?(g) }
-importer = Gman::Importer.new(domains)
-
-importer.logger.info "Starting with #{importer.domains.count} domains"
-
-importer.domains.list.each_value do |d|
-  d.map! { |domain| Gman.new(domain).to_s }
-  d.map! { |domain| importer.normalize_domain(domain) }
-end
-
-count = importer.domains.domains.count
-importer.logger.info "Filtered down to #{count} normalized domains"
-
-missing = {}
-importer.domains.list.each do |g, usagovdomains|
-  next unless importer.current.list[g]
-
-  missing[g] = importer.current.list[g] - usagovdomains
-end
-
-missing.reject! { |_key, value| value.empty? }
-
-count = missing.values.count
-importer.logger.info "Found #{count} domains not on the USA.gov list"
-puts "Here's the list of missing domains:"
-puts YAML.dump(missing)
-
-domains = importer.domains.domains
-domains = domains.group_by do |domain|
-  importer.valid_domain?(domain, skip_dupe: true)
-end
-domains.delete(true)
-domains.delete(false)
-domains.delete('locality')
-
-count = domains.values.flatten.count
-importer.logger.info "Calling out #{count} rejected domains"
-
-puts 'Here are the rejected domains and why they were rejected:'
-puts YAML.dump(domains)

data/script/release

@@ -1,38 +0,0 @@
-#!/bin/sh
-# Tag and push a release.
-
-set -e
-
-# Make sure we're in the project root.
-
-cd $(dirname "$0")/..
-
-# Build a new gem archive.
-
-rm -rf gman-*.gem
-gem build -q gman.gemspec
-
-# Make sure we're on the master branch.
-
-(git branch | grep -q '* master') || {
-  echo "Only release from the master branch."
-  exit 1
-}
-
-# Figure out what version we're releasing.
-
-tag=v`ls gman-*.gem | sed 's/^gman-\(.*\)\.gem$/\1/'`
-
-# Make sure we haven't released this version before.
-
-git fetch -t origin
-
-(git tag -l | grep -q "$tag") && {
-  echo "Whoops, there's already a '${tag}' tag."
-  exit 1
-}
-
-# Tag it and bag it.
-
-gem push gman-*.gem && git tag "$tag" &&
-  git push origin master && git push origin "$tag"

data/script/validate-domains

@@ -1,34 +0,0 @@
-#!/usr/bin/env ruby
-
-# ! /usr/bin/env ruby
-# frozen_string_literal: true
-
-#
-# Add one or more domains to a given group, running the standard import checks
-#
-# Usage: script/add [GROUP] [DOMAIN(S)]
-
-require './lib/gman/importer'
-require 'parallel'
-
-importer = Gman::Importer.new({})
-options = { skip_dupe: true, skip_resolve: false }
-list_path = File.expand_path '../config/domains.txt', __dir__
-
-importer.logger.info "Starting list: #{Gman::DomainList.current.count} domains"
-
-Gman.list.to_h.values.shuffle.each do |domains|
-  # next if ['non-us gov', 'non-us mil', 'US Federal'].include?(group)
-  
-  Parallel.each(domains, progress: "Validating") do |domain|
-    next if domain.start_with?("!")
-    next if importer.valid_domain?(domain, options)
-
-    importer.logger.warn "#{domain} is not valid, removing from list"
-    list = File.read(list_path)
-    list.gsub!(/^#{Regexp.escape(domain)}$\n/, '')
-    File.write list_path, list
-  end
-end
-
-importer.logger.info "Ending list: #{Gman::DomainList.current.count} domains"

data/script/vendor

@@ -1,13 +0,0 @@
-#!/bin/bash
-#Runs all vendor scripts except script/vendor-nl
-
-for file in script/vendor-*; do
-  if [ "$file" != "script/vendor-nl" ]; then
-    echo "*************************************"
-    echo "Vendoring $file"
-    echo "*************************************"
-    bundle exec "$file"
-  fi
-done
-
-bundle exec script/alphabetize

data/script/vendor-federal-de

@@ -1,14 +0,0 @@
-#! /usr/bin/env ruby
-# frozen_string_literal: true
-
-require 'csv'
-require 'open-uri'
-require './lib/gman'
-
-url = 'https://raw.githubusercontent.com/robbi5/german-gov-domains/master/data/domains.csv'
-
-domains = URI.open(url).read.encode('UTF-8')
-domains = CSV.parse(domains, headers: true)
-domains = domains.map { |row| row['Domain Name'] }
-
-Gman::Importer.new('German Federal' => domains).import

data/script/vendor-gov-list

@@ -1,8 +0,0 @@
-#!/bin/sh
-#
-# Vendors the full list of US .gov domains from https://github.com/GSA/data
-# Usage: script/vendor-gov-list
-
-# Vendor the last file in the dotgov-domains folder that ends in `-full.csv`
-wget https://raw.githubusercontent.com/cisagov/dotgov-data/main/current-full.csv -O ./config/vendor/dotgovs.csv
-

data/script/vendor-public-suffix

@@ -1,28 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-# Propagates an initial list of best-guess government domains
-
-require 'public_suffix'
-require 'yaml'
-require_relative '../lib/gman'
-
-# https://gist.github.com/benbalter/6147066
-REGEX = /(\.g[ou]{1,2}(v|b|vt)|\.mil|\.gc|\.fed)(\.[a-z]{2})?$/i.freeze
-
-domains = []
-PublicSuffix::List.default.each do |rule|
-  domain = nil
-
-  if rule.parts.length == 1
-    domain = rule.parts.first if REGEX.match?(".#{rule.value}")
-  elsif REGEX.match?(".#{rule.value}")
-    domain = rule.parts.pop(2).join('.')
-  end
-
-  domains.push domain unless domain.nil? || domains.include?(domain)
-end
-
-# NOTE: We want to skip resolution here, because a domain like `gov.sv` may be
-# a valid TLD, not have any top-level sites, and we'd still want it listed
-Gman::Importer.new('non-us gov' => domains).import(skip_resolve: true)

data/script/vendor-swot

@@ -1,43 +0,0 @@
-#! /usr/bin/env ruby
-# frozen_string_literal: true
-
-#
-# Vendors the Swot-maintained list of adademic domains into config/academic.txt
-# Source: https://github.com/leereilly/swot/
-#
-# Usage: script/vendor-swot
-#
-# Will automatically fetch latest version of the list and merge
-# You can check for changes and commit via `git status`
-#
-# It's also probably a good idea to run `script/ci-build` for good measure
-#
-# Note: We do this, because as a bajillion individual files, Swot takes up 30MB
-
-require 'gman'
-require 'swot'
-
-# Generate array of all Swot domains
-domains = Swot.all_domains
-domains << Swot::ACADEMIC_TLDS
-
-# Init the importer, builiding a DomainList
-group = "Academic domains vendored from Swot v#{Swot::VERSION}"
-hash = { group => domains }
-
-importer = Gman::Importer.new(hash)
-importer.logger.info "Importing from Swot v#{Swot::VERSION}"
-importer.logger.info "Found #{domains.count} academic domains"
-
-domain_list = importer.domain_list
-domain_list.path = Gman.academic_list_path
-
-# Cleanup and write
-# Note: we're not using the import method, as that assume's we're writing the
-# government domain list and would use Swot to ensure domains aren't academic
-importer.send :normalize_domains!
-domain_list.data[group] << Swot::BLACKLIST.map { |domain| "!#{domain}" }
-domain_list.data[group] = domain_list.data[group].flatten
-domain_list.write
-
-importer.logger.info "Vendored #{importer.domain_list.count} academic domains."

data/script/vendor-us

@@ -1,42 +0,0 @@
-#! /usr/bin/env ruby
-# frozen_string_literal: true
-
-#
-# Vendors the USA.gov-maintained list of US domains into domains.txt
-# Source: https://github.com/GSA-OCSIT/govt-urls
-#
-# Usage: script/vendor-us
-#
-# Will automatically fetch latest version of the list and merge
-# You can check for changes and commit via `git status`
-#
-# It's also probably a good idea to run `script/ci-build` for good measure
-
-require './lib/gman'
-require 'open-uri'
-require 'csv'
-
-path = File.expand_path('./vendor-us-tmp.csv')
-blacklist = %w[usagovQUASI usagovFEDgov]
-source = 'https://raw.githubusercontent.com/GSA/govt-urls/main/1_govt_urls_full.csv'
-domains = {}
-
-begin
-  raw = URI.open(source).read
-  File.write(path, raw)
-  data = CSV.table(path)
-
-  data.each do |domain|
-    next if domain[:type_of_government] == 'Quasigovernmental'
-
-    group = "US #{domain[:type_of_government]}"
-    group += " (#{domain[:state]})" if domain[:type_of_government] != 'Federal' && domain[:state]
-    domains[group] ||= []
-    domains[group] << domain[:domain_name]
-  end
-
-  domains.reject! { |g, _| blacklist.include?(g) }
-  Gman::Importer.new(domains).import
-ensure
-  File.delete(path)
-end

data/spec/fixtures/domains.txt

@@ -1,4 +0,0 @@
-// foo
-bar.gov
-baz.net
-!mail.bar.gov

data/spec/fixtures/obama.txt

@@ -1,5 +0,0 @@
-barry@dcpchicago.org
-prof.obama@uchicago.edu
-mr.senator@obama.senate.gov
-president@whitehouse.gov
-commander.in.chief@us.army.mil

data/spec/gman/bin_spec.rb

@@ -1,101 +0,0 @@
-# frozen_string_literal: true
-
-RSpec.describe 'Gman bin' do
-  let(:domain) { 'whitehouse.gov' }
-  let(:args) { [domain] }
-  let(:command) { 'gman' }
-  let(:bin_path) do
-    File.expand_path "../../bin/#{command}", File.dirname(__FILE__)
-  end
-  let(:response_parts) { Open3.capture2e('bundle', 'exec', bin_path, *args) }
-  let(:output) { response_parts[0] }
-  let(:status) { response_parts[1] }
-  let(:exit_code) { status.exitstatus }
-
-  context 'a valid domain' do
-    it 'parses the domain' do
-      expect(output).to match('Domain  : whitehouse.gov')
-    end
-
-    it "knows it's valid" do
-      expect(output).to match('Valid government domain')
-      expect(exit_code).to be(0)
-    end
-
-    it 'knows the type' do
-      expect(output).to match(/federal/i)
-    end
-
-    it 'knows the agency' do
-      expect(output).to match('Executive Office of the President')
-    end
-
-    it 'knows the country' do
-      expect(output).to match('United States')
-    end
-
-    it 'knows the city' do
-      expect(output).to match('Washington')
-    end
-
-    it 'knows the state' do
-      expect(output).to match('DC')
-    end
-
-    it 'colors by default' do
-      expect(output).to match(/\e\[32m/)
-    end
-
-    context 'with colorization disabled' do
-      let(:args) { [domain, '--no-color'] }
-
-      it "doesn't color" do
-        expect(output).not_to match(/\e\[32m/)
-      end
-    end
-  end
-
-  context 'with no args' do
-    let(:args) { [] }
-
-    it 'displays the help text' do
-      expect(output).to match('USAGE')
-    end
-  end
-
-  context 'an invalid domain' do
-    let(:domain) { 'foo.invalid' }
-
-    it 'knows the domain is invalid' do
-      expect(output).to match('Invalid domain')
-      expect(exit_code).to be(1)
-    end
-  end
-
-  context 'a non-government domain' do
-    let(:domain) { 'github.com' }
-
-    it "knows it's not a government domain" do
-      expect(output).to match('Not a government domain')
-      expect(exit_code).to be(1)
-    end
-  end
-
-  context 'filtering' do
-    let(:command) { 'gman_filter' }
-    let(:txt_path) do
-      File.expand_path '../fixtures/obama.txt', File.dirname(__FILE__)
-    end
-    let(:args) { [txt_path] }
-
-    it 'returns only government domains' do
-      expected = <<~EXPECTED
-        mr.senator@obama.senate.gov
-        president@whitehouse.gov
-        commander.in.chief@us.army.mil
-      EXPECTED
-
-      expect(output).to eql(expected)
-    end
-  end
-end

data/spec/gman/country_code_spec.rb

@@ -1,39 +0,0 @@
-# frozen_string_literal: true
-
-RSpec.describe 'Gman Country Codes' do
-  {
-    'whitehouse.gov' => 'United States of America',
-    'foo.gov.uk' => 'United Kingdom of Great Britain and Northern Ireland',
-    'army.mil' => 'United States of America',
-    'foo.gc.ca' => 'Canada',
-    'foo.eu' => nil
-  }.each do |domain, expected_country|
-    context "given #{domain.inspect}" do
-      subject { Gman.new(domain) }
-
-      let(:country) { subject.country }
-
-      it 'knows the country' do
-        if expected_country.nil?
-          expect(country).to be_nil
-        else
-          expect(country.name).to eql(expected_country)
-        end
-      end
-
-      it 'knows the alpha2' do
-        expected = case expected_country
-                   when 'United States of America'
-                     'us'
-                   when 'Canada'
-                     'ca'
-                   when 'United Kingdom of Great Britain and Northern Ireland'
-                     'gb'
-                   else
-                     'eu'
-                   end
-        expect(subject.alpha2).to eql(expected)
-      end
-    end
-  end
-end

data/spec/gman/domain_list_spec.rb

@@ -1,110 +0,0 @@
-# frozen_string_literal: true
-
-RSpec.describe Gman::DomainList do
-  let(:data) { subject.data }
-  let(:canada) { data['Canada municipal'] }
-
-  %i[path contents data].each do |type|
-    context "when initialized by #{type}" do
-      subject do
-        case type
-        when :path
-          described_class.new(path: Gman.list_path)
-        when :contents
-          contents = File.read(Gman.list_path)
-          described_class.new(contents: contents)
-        when :data
-          data = described_class.new(path: Gman.list_path).to_h
-          described_class.new(data: data)
-        end
-      end
-
-      it 'stores the init var' do
-        expect(subject.send(type)).not_to be_nil
-      end
-
-      it 'returns the domain data' do
-        expect(data).to have_key('Canada federal')
-        expect(data.values.flatten).to include('gov')
-      end
-
-      it 'returns the list contents' do
-        expect(subject.contents).to match(/^gov$/)
-      end
-
-      it 'knows the list path' do
-        expect(subject.path).to eql(Gman.list_path)
-      end
-
-      it 'returns the PublicSuffix list' do
-        expect(subject.public_suffix_list).to be_a(PublicSuffix::List)
-      end
-
-      it 'knows if a domain is valid' do
-        expect(subject.valid?('whitehouse.gov')).to be(true)
-      end
-
-      it 'knows if a domain is invalid' do
-        expect(subject.valid?('example.com')).to be(false)
-      end
-
-      it 'returns the domain groups' do
-        expect(subject.groups).to include('Canada federal')
-      end
-
-      it 'returns the domains' do
-        expect(subject.domains).to include('gov')
-      end
-
-      it 'returns the domain count' do
-        expect(subject.count).to be_a(Integer)
-        expect(subject.count).to be > 100
-      end
-
-      it 'alphabetizes the list' do
-        canada.shuffle!
-        expect(canada.first).not_to eql('100milehouse.com')
-        subject.alphabetize
-        expect(canada.first).to eql('100milehouse.com')
-      end
-
-      it 'outputs public suffix format' do
-        expect(subject.to_s).to match("// Canada federal\ncanada.ca\n")
-      end
-
-      it "finds a domain's parent" do
-        expect(subject.parent_domain('foo.gov.uk')).to eql('gov.uk')
-      end
-
-      context 'with the list path stubbed' do
-        let(:stubbed_file_contents) { File.read(stubbed_list_path) }
-
-        before do
-          subject.instance_variable_set(:@path, stubbed_list_path)
-        end
-
-        context 'with list data stubbed' do
-          before do
-            subject.data = { 'foo' => ['!mail.bar.gov', 'bar.gov', 'baz.net'] }
-          end
-
-          context 'alphabetizing' do
-            before { subject.alphabetize }
-
-            it 'puts exceptions last' do
-              expect(subject.data['foo'].last).to eql('!mail.bar.gov')
-            end
-          end
-
-          context 'writing' do
-            before { subject.write }
-
-            it 'writes the contents' do
-              expect(stubbed_file_contents).to match("// foo\nbar.gov\nbaz.net")
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/spec/gman/domains_spec.rb

@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-
-RSpec.describe 'Gman domains' do
-  let(:resolve_domains?) { ENV['GMAN_RESOLVE_DOMAINS'] == 'true' }
-  let(:importer) { Gman::Importer.new({}) }
-  let(:options) { { skip_dupe: true, skip_resolve: !resolve_domains? } }
-
-  Gman.list.to_h.each do |group, domains|
-    next if ['non-us gov', 'non-us mil', 'US Federal'].include?(group)
-
-    context "the #{group} group" do
-      it 'only contains valid domains' do
-        invalid_domains = []
-
-        Parallel.each(domains, in_threads: 4) do |domain|
-          next if importer.valid_domain?(domain, options)
-
-          invalid_domains.push domain
-        end
-
-        expect(invalid_domains).to be_empty
-      end
-    end
-  end
-end