gman 7.0.4 → 7.0.6

data/gman.gemspec

@@ -18,17 +18,16 @@ Gem::Specification.new do |s|
   s.licenses = ['MIT']
 
   s.files         = `git ls-files`.split("\n")
-  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map do |f|
     File.basename(f)
   end
 
   s.require_paths = ['lib']
-  s.required_ruby_version = '~> 2.5'
+  s.required_ruby_version = '>= 2.5', '< 4.0'
 
   s.add_dependency('colored', '~> 1.2')
   s.add_dependency('iso_country_codes', '~> 0.6')
-  s.add_dependency('naughty_or_nice', '= 2.1.1')
+  s.add_dependency('naughty_or_nice', '>= 2.1.1')
   s.add_dependency('public_suffix', '>= 3.0')
 
   s.add_development_dependency('addressable', '~> 2.3')
@@ -39,6 +38,8 @@ Gem::Specification.new do |s|
   s.add_development_dependency('rubocop', '~> 1.0')
   s.add_development_dependency('rubocop-performance', '~> 1.5')
   s.add_development_dependency('rubocop-rspec', '~> 2.0')
-  s.add_development_dependency('ruby-prof', '~> 0.15')
+  s.add_development_dependency('ruby-prof', '~> 1.4')
+  s.add_development_dependency('ruby-progressbar', '~> 1.10')
   s.add_development_dependency('swot', '~> 1.0')
+  s.metadata['rubygems_mfa_required'] = 'true'
 end

data/lib/gman/identifier.rb

@@ -26,7 +26,7 @@ class Gman
   def_hash_delegator :dotgov_listing, :Agency
   def_hash_delegator :dotgov_listing, :Organization
   def_hash_delegator :dotgov_listing, :City
-  def_hash_delegator :dotgov_listing, :"Domain Type"
+  def_hash_delegator :dotgov_listing, :'Domain Type'
   private :domain_type
 
   def type
@@ -60,7 +60,7 @@ class Gman
   def federal?
     return false unless dotgov_listing
 
-    domain_type =~ /^Federal Agency/i
+    domain_type =~ /^Federal/i
   end
 
   def city?
@@ -87,7 +87,7 @@ class Gman
     if matches
       matches[1] == 'state'
     elsif dotgov_listing
-      domain_type == 'State/Local Govt'
+      domain_type == 'State/Local Govt' || domain_type == 'State'
     else
       false
     end
@@ -108,14 +108,14 @@ class Gman
   private
 
   def list_category
-    @list_category ||= begin
-      match = Gman.list.public_suffix_list.find(domain.to_s)
-      return unless match
+    return @list_category if defined?(@list_category)
 
-      regex = %r{// ([^\n]+)\n?[^/]*\n#{Regexp.escape(match.value)}\n}im
-      matches = Gman.list.contents.match(regex)
-      matches[1] if matches
-    end
+    match = Gman.list.public_suffix_list.find(domain.to_s)
+    return @list_category = nil unless match
+
+    regex = %r{// ([^\n]+)\n?[^/]*\n#{Regexp.escape(match.value)}\n}im
+    matches = Gman.list.contents.match(regex)
+    @list_category = matches ? matches[1] : nil
   end
 
   def matches

data/lib/gman/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class Gman
-  VERSION = '7.0.4'
+  VERSION = '7.0.6'
 end

data/lib/gman.rb

@@ -43,12 +43,11 @@ class Gman
   #
   # Returns boolean true if a government domain
   def valid?
-    @valid ||= begin
-      return false unless valid_domain?
-      return false if academic?
+    return @valid if defined?(@valid)
 
-      locality? || public_suffix_valid?
-    end
+    @valid = false unless valid_domain?
+    @valid = false if academic?
+    @valid ||= locality? || public_suffix_valid?
   end
 
   def locality?

data/script/profile

@@ -8,7 +8,7 @@ require './lib/gman'
 # without pre-loading the Gman list for an accurate benchmark
 count = (ARGV[0] || 100).to_i
 domains = File.readlines('./config/domains.txt')
-domains = domains.select { |l| l =~ /^[a-z0-9]/i }
+domains = domains.grep(/^[a-z0-9]/i)
 domains = domains.sample(count)
 
 RubyProf.start

data/script/prune

@@ -10,7 +10,7 @@ require_relative '../lib/gman/domain_list'
 domains = ARGV
 domains = domains.clone.map { |d| d.delete ',' }
 
-list = File.open('./config/domains.txt').read
+list = File.read('./config/domains.txt')
 puts "Starting list: #{Gman::DomainList.current.count} domains"
 
 domains.each do |domain|

data/script/reconcile-us

@@ -26,7 +26,7 @@ data.each do |row|
     group = row
     domains[group] = []
   else
-    domains[group].push row.sub("\.\t", '').strip
+    domains[group].push row.sub(".\t", '').strip
   end
 end
 

data/script/validate-domains

@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+
+# ! /usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# Add one or more domains to a given group, running the standard import checks
+#
+# Usage: script/add [GROUP] [DOMAIN(S)]
+
+require './lib/gman/importer'
+require 'parallel'
+
+importer = Gman::Importer.new({})
+options = { skip_dupe: true, skip_resolve: false }
+list_path = File.expand_path '../config/domains.txt', __dir__
+
+importer.logger.info "Starting list: #{Gman::DomainList.current.count} domains"
+
+Gman.list.to_h.values.shuffle.each do |domains|
+  # next if ['non-us gov', 'non-us mil', 'US Federal'].include?(group)
+  
+  Parallel.each(domains, progress: "Validating") do |domain|
+    next if domain.start_with?("!")
+    next if importer.valid_domain?(domain, options)
+
+    importer.logger.warn "#{domain} is not valid, removing from list"
+    list = File.read(list_path)
+    list.gsub!(/^#{Regexp.escape(domain)}$\n/, '')
+    File.write list_path, list
+  end
+end
+
+importer.logger.info "Ending list: #{Gman::DomainList.current.count} domains"

data/script/vendor

@@ -10,4 +10,4 @@ for file in script/vendor-*; do
   fi
 done
 
-script/alphabetize
+bundle exec script/alphabetize

data/script/vendor-gov-list

@@ -3,15 +3,6 @@
 # Vendors the full list of US .gov domains from https://github.com/GSA/data
 # Usage: script/vendor-gov-list
 
-# Set up
-mkdir tmp
-rm -Rf tmp/gsa-data
-
 # Vendor the last file in the dotgov-domains folder that ends in `-full.csv`
-git clone https://github.com/GSA/data tmp/gsa-data
-pattern="tmp/gsa-data/dotgov-domains/*-full.csv"
-files=( $pattern )
-cp -f "${files[@]:(-1)}" config/vendor/dotgovs.csv
+wget https://raw.githubusercontent.com/cisagov/dotgov-data/main/current-full.csv -O ./config/vendor/dotgovs.csv
 
-# Clean up
-rm -Rf tmp/gsa-data

data/script/vendor-us

@@ -14,25 +14,29 @@
 
 require './lib/gman'
 require 'open-uri'
+require 'csv'
 
+path = File.expand_path('./vendor-us-tmp.csv')
 blacklist = %w[usagovQUASI usagovFEDgov]
-source = 'https://raw.githubusercontent.com/GSA/govt-urls/master/government-urls-hierarchical-list.txt'
+source = 'https://raw.githubusercontent.com/GSA/govt-urls/main/1_govt_urls_full.csv'
+domains = {}
 
-data = URI.open(source).read
-data = data.split('_' * 74)
-data = data.last.strip
-data = data.split(/\r?\n/).reject(&:empty?)
+begin
+  raw = URI.open(source).read
+  File.write(path, raw)
+  data = CSV.table(path)
 
-domains = {}
-group = ''
-data.each do |row|
-  if /^\w/.match?(row)
-    group = row
-    domains[group] = []
-  else
-    domains[group].push row.sub("\.\t", '').strip
+  data.each do |domain|
+    next if domain[:type_of_government] == 'Quasigovernmental'
+
+    group = "US #{domain[:type_of_government]}"
+    group += " (#{domain[:state]})" if domain[:type_of_government] != 'Federal' && domain[:state]
+    domains[group] ||= []
+    domains[group] << domain[:domain_name]
   end
-end
 
-domains.reject! { |g, _| blacklist.include?(g) }
-Gman::Importer.new(domains).import
+  domains.reject! { |g, _| blacklist.include?(g) }
+  Gman::Importer.new(domains).import
+ensure
+  File.delete(path)
+end

data/spec/gman/domain_list_spec.rb

@@ -69,7 +69,7 @@ RSpec.describe Gman::DomainList do
       end
 
       it 'outputs public suffix format' do
-        expect(subject.to_s).to match("// Canada federal\ncanada\.ca\n")
+        expect(subject.to_s).to match("// Canada federal\ncanada.ca\n")
       end
 
       it "finds a domain's parent" do
@@ -80,7 +80,7 @@ RSpec.describe Gman::DomainList do
         let(:stubbed_file_contents) { File.read(stubbed_list_path) }
 
         before do
-          subject.instance_variable_set('@path', stubbed_list_path)
+          subject.instance_variable_set(:@path, stubbed_list_path)
         end
 
         context 'with list data stubbed' do

data/spec/gman/identifier_spec.rb

@@ -137,7 +137,7 @@ RSpec.describe 'Gman identifier' do
       end
 
       context 'a county .gov' do
-        let(:domain) { 'ALLEGHENYCOUNTYPA.GOV' }
+        let(:domain) { '211DUPAGE.GOV' }
 
         it "knows it's a county" do
           expect(subject).to be_a_county
@@ -161,11 +161,11 @@ RSpec.describe 'Gman identifier' do
         end
 
         it 'knows the state' do
-          expect(subject.state).to eql('PA')
+          expect(subject.state).to eql('IL')
         end
 
         it 'knows the city' do
-          expect(subject.city).to eql('Pittsburgh')
+          expect(subject.city).to eql('Wheaton')
         end
       end
 
@@ -203,8 +203,8 @@ RSpec.describe 'Gman identifier' do
   context "determining a domain's type" do
     {
       unknown: 'cityofperu.org',
-      "Canada municipal": 'acme.ca',
-      "Canada federal": 'canada.ca'
+      'Canada municipal': 'acme.ca',
+      'Canada federal': 'canada.ca'
     }.each do |expected, domain|
       context "Given the #{domain} domain" do
         let(:domain) { domain }

data/spec/gman/importer_spec.rb

@@ -9,7 +9,7 @@ RSpec.describe Gman::Importer do
   let(:domain_list) { subject.domain_list }
 
   before do
-    subject.instance_variable_set '@logger', logger
+    subject.instance_variable_set :@logger, logger
   end
 
   it 'inits the domain list' do
@@ -68,7 +68,7 @@ RSpec.describe Gman::Importer do
     let(:stubbed_list) { Gman::DomainList.new(path: stubbed_list_path) }
     let(:stubbed_file_contents) { File.read(stubbed_list_path) }
 
-    before { subject.instance_variable_set '@current', stubbed_list }
+    before { subject.instance_variable_set :@current, stubbed_list }
 
     context 'writing' do
       before { @current = subject.current.to_s }

data/spec/gman_spec.rb

@@ -18,7 +18,7 @@ RSpec.describe Gman do
 
   context 'invalid domains' do
     ['foo.bar.com', 'bar@foo.biz', 'http://www.foo.biz',
-     'foo.uk', 'gov', 'foo@k12.champaign.il.us', 'foo@kii.gov.by',
+     'foo.uk', 'gov', 'foo@k12.champaign.il.us', # 'foo@kii.gov.by',
      'foo', '', nil, ' ', 'foo.city.il.us', 'foo.ci.il.us',
      'foo.zx.us', 'foo@mail.gov.ua', 'foo@gwu.edu'].each do |domain|
       subject { described_class.new(domain) }

data/spec/spec_helper.rb

@@ -24,7 +24,7 @@ def stubbed_list_path
 end
 
 def with_env(key, value)
-  old_env = ENV[key]
+  old_env = ENV.fetch(key, nil)
   ENV[key] = value
   yield
   ENV[key] = old_env

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gman
 version: !ruby/object:Gem::Version
-  version: 7.0.4
+  version: 7.0.6
 platform: ruby
 authors:
 - Ben Balter
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-11-13 00:00:00.000000000 Z
+date: 2022-12-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: colored
@@ -42,14 +42,14 @@ dependencies:
   name: naughty_or_nice
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.1.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.1.1
 - !ruby/object:Gem::Dependency
@@ -184,14 +184,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.15'
+        version: '1.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.15'
+        version: '1.10'
 - !ruby/object:Gem::Dependency
   name: swot
   requirement: !ruby/object:Gem::Requirement
@@ -220,17 +234,21 @@ files:
 - ".github/ISSUE_TEMPLATE/bug_report.md"
 - ".github/ISSUE_TEMPLATE/feature_request.md"
 - ".github/config.yml"
+- ".github/dependabot.yml"
 - ".github/funding.yml"
 - ".github/no-response.yml"
 - ".github/release-drafter.yml"
 - ".github/settings.yml"
 - ".github/stale.yml"
+- ".github/workflows/ci.yml"
+- ".github/workflows/clean.yml"
+- ".github/workflows/codeql-analysis.yml"
+- ".github/workflows/validate.yml"
+- ".github/workflows/vendor.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
 - ".rubocop_todo.yml"
-- ".ruby-version"
-- ".travis.yml"
 - Gemfile
 - LICENSE
 - bin/gman
@@ -261,13 +279,11 @@ files:
 - script/prune
 - script/reconcile-us
 - script/release
+- script/validate-domains
 - script/vendor
 - script/vendor-federal-de
 - script/vendor-gov-list
-- script/vendor-municipal-de
-- script/vendor-nl
 - script/vendor-public-suffix
-- script/vendor-se
 - script/vendor-swot
 - script/vendor-us
 - spec/fixtures/domains.txt
@@ -284,35 +300,28 @@ files:
 homepage: https://github.com/benbalter/gman
 licenses:
 - MIT
-metadata: {}
+metadata:
+  rubygems_mfa_required: 'true'
 post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - "~>"
+  - - ">="
     - !ruby/object:Gem::Version
       version: '2.5'
+  - - "<"
+    - !ruby/object:Gem::Version
+      version: '4.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.2.33
 signing_key:
 specification_version: 4
 summary: Check if a given domain or email address belong to a governemnt entity
-test_files:
-- spec/fixtures/domains.txt
-- spec/fixtures/obama.txt
-- spec/gman/bin_spec.rb
-- spec/gman/country_code_spec.rb
-- spec/gman/domain_list_spec.rb
-- spec/gman/domains_spec.rb
-- spec/gman/identifier_spec.rb
-- spec/gman/importer_spec.rb
-- spec/gman/locality_spec.rb
-- spec/gman_spec.rb
-- spec/spec_helper.rb
+test_files: []

data/.ruby-version

@@ -1 +0,0 @@
-2.6.6

data/.travis.yml

@@ -1,4 +0,0 @@
-langauage: ruby
-script: "script/cibuild"
-sudo: false
-cache: bundler

data/script/vendor-municipal-de

@@ -1,23 +0,0 @@
-#! /usr/bin/env ruby
-# frozen_string_literal: true
-
-require 'csv'
-require 'open-uri'
-require './lib/gman'
-
-url = 'http://www.mik.nrw.de/nc/themen-aufgaben/kommunales/kommunale-adressen.html?tx_szkommunaldb_pi1%5Bexport%5D=csv'
-
-csv = URI.open(url).read.force_encoding('iso-8859-1').encode('UTF-8')
-
-# For some reason, the header row is actually the last row
-# Pop the last line off the file and prepend it at the begining
-# So that when we pass it to CSV it detects the headers properly
-lines = csv.split("\n")
-lines.unshift lines.pop
-csv = lines.join("\n")
-
-# Load municipal domains
-data = CSV.parse(csv, headers: true, col_sep: ';')
-domains = data.map { |row| row['Internet'] }
-
-Gman::Importer.new('German Municipalities' => domains).import

data/script/vendor-nl

@@ -1,21 +0,0 @@
-#! /usr/bin/env ruby
-# frozen_string_literal: true
-
-# See https://github.com/github/government.github.com/pull/367#issuecomment-102108763
-
-require 'fileutils'
-require './lib/gman'
-
-FileUtils.rm_rf('almanak.overheid.nl')
-commands = [
-  "wget -q -r -nc -np https://almanak.overheid.nl/
-  grep @ -rI almanak.overheid.nl/",
-  'cut -f 2 -d @',
-  "cut -f 1 -d '\"'",
-  'grep \\.nl$',
-  'sort',
-  'uniq'
-]
-domains = system commands.join('|')
-
-Gman::Importer.new('Netherlands' => domains.split("\n")).import

data/script/vendor-se

@@ -1,21 +0,0 @@
-#! /usr/bin/env ruby
-# frozen_string_literal: true
-
-require 'mechanize'
-require 'csv'
-require './lib/gman'
-
-url = 'http://www.myndighetsregistret.scb.se/Myndighet.aspx'
-agent = Mechanize.new
-page = agent.get(url)
-form = page.forms.first
-form.radiobuttons.find { |r| r.value = 'Textfil' }.check
-submit_button = form.buttons.find { |b| b.type == 'submit' }
-response = agent.submit(form, submit_button)
-
-rows = CSV.parse(response.content, headers: true, col_sep: "\t")
-domains = rows.map do |row|
-  row['Webbadress'] unless /UNIVERSITET/.match?(row['Namn'])
-end
-
-Gman::Importer.new('Swedish Administrative Authorities' => domains).import