gman 7.0.2 → 7.0.3

data/lib/gman/importer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Utility functions for parsing and manipulating public-suffix domain lists
 # Only used in development and not loaded by default
 require 'yaml'
@@ -13,7 +15,7 @@ class Gman
     attr_accessor :domain_list
 
     # Known false positives from vendored lists
-    BLACKLIST = %w(
+    BLACKLIST = %w[
       business.centurytel.net
       chesnee.net
       citlink.net
@@ -39,23 +41,24 @@ class Gman
       wctc.net
       webconnections.net
       webpages.charter.net
-    ).freeze
+    ].freeze
 
     REGEX_CHECKS = {
-      'home. regex'     => /^home\./,
-      'user. regex'     => /^users?\./,
-      'sites. regex'    => /^sites?\./,
-      'weebly'          => /weebly\.com$/,
-      'wordpress'       => /wordpress\.com$/,
-      'govoffice'       => /govoffice\d?\.com$/,
-      'homestead'       => /homestead\.com$/,
-      'wix.com'         => /wix\.com$/,
-      'blogspot.com'    => /blogspot\.com$/,
-      'tripod.com'      => /tripod\.com$/,
+      'home. regex' => /^home\./,
+      'user. regex' => /^users?\./,
+      'sites. regex' => /^sites?\./,
+      'weebly' => /weebly\.com$/,
+      'wordpress' => /wordpress\.com$/,
+      'govoffice' => /govoffice\d?\.com$/,
+      'homestead' => /homestead\.com$/,
+      'wix.com' => /wix\.com$/,
+      'blogspot.com' => /blogspot\.com$/,
+      'tripod.com' => /tripod\.com$/,
       'squarespace.com' => /squarespace\.com$/,
-      'github.io'       => /github\.io$/,
-      'tumblr'          => /tumblr\.com$/,
-      'locality'        => Gman::Locality::REGEX
+      'github.io' => /github\.io$/,
+      'tumblr' => /tumblr\.com$/,
+      'locality' => Gman::Locality::REGEX,
+      'french edu' => /^ac-.*?\.fr/
     }.freeze
 
     def initialize(domains)
@@ -75,6 +78,7 @@ class Gman
       return false if !options[:skip_dupe] && !ensure_not_dupe(domain)
       return false unless ensure_valid(domain)
       return false if !options[:skip_resolve] && !ensure_resolves(domain)
+
       true
     end
 
@@ -82,6 +86,7 @@ class Gman
     # rather than a bool and silence log output
     def reject(domain, reason)
       return reason if ENV['RECONCILING']
+
       logger.info "👎 `#{domain}`: #{reason}"
       false
     end
@@ -102,13 +107,14 @@ class Gman
     end
 
     def resolver
-      @resolver ||= Resolv::DNS.new(nameserver: ['8.8.8.8', '8.8.4.4'])
+      @resolver ||= Resolv::DNS.new(nameserver: ['1.1.1.1', '8.8.8.8'])
     end
 
     # Verifies that the given domain has an MX record, and thus is valid
     def domain_resolves?(domain)
       domain = Addressable::URI.new(host: domain).normalize.host
       return true if ip?(domain)
+
       returns_record?(domain, 'NS') || returns_record?(domain, 'MX')
     end
 
@@ -123,6 +129,7 @@ class Gman
 
     def ensure_valid(domain)
       return false if domain.empty?
+
       if BLACKLIST.include?(domain)
         reject(domain, 'blacklist')
       elsif !PublicSuffix.valid?("foo.#{domain}")
@@ -136,11 +143,13 @@ class Gman
 
     def ensure_resolves(domain)
       return reject(domain, 'unresolvable') unless domain_resolves?(domain)
+
       true
     end
 
     def ensure_not_dupe(domain)
       return true unless dupe?(domain)
+
       if current.domains.include?(domain)
         reject(domain, 'duplicate')
       else
@@ -154,14 +163,14 @@ class Gman
     end
 
     def normalize_domains!
-      domain_list.to_h.each do |_group, domains|
+      domain_list.to_h.each_value do |domains|
         domains.map! { |domain| normalize_domain(domain) }
         domains.uniq!
       end
     end
 
     def ensure_validity!(options = {})
-      domain_list.data.each do |_group, domains|
+      domain_list.data.each_value do |domains|
         domains.select! { |domain| valid_domain?(domain, options) }
       end
     end

data/lib/gman/locality.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class Gman
   # Second level .us domains for states and locality
   # See http://en.wikipedia.org/wiki/.us
@@ -12,18 +14,18 @@ class Gman
   #  * k12.il.us
   #  * ci.foo.zx.us
   class Locality
-    AFFINITY_NAMESPACES = %w(state dst cog).freeze
+    AFFINITY_NAMESPACES = %w[state dst cog].freeze
 
-    STATES = %w(
+    STATES = %w[
       ak al ar az ca co ct dc de fl ga hi ia id il in ks ky
       la ma md me mi mn mo ms mt nc nd ne nh nj nm nv ny oh
       ok or pa ri sc sd tn tx um ut va vt wa wi wv wy
-    ).freeze
+    ].freeze
 
-    LOCALITY_DOMAINS = %w(
+    LOCALITY_DOMAINS = %w[
       ci co borough boro city county
       parish town twp vi vil village
-    ).freeze
+    ].freeze
 
     REGEX = /
       (
@@ -31,7 +33,7 @@ class Gman
       |
         (#{Regexp.union(LOCALITY_DOMAINS)})\.[a-z-]+
       )\.(#{Regexp.union(STATES)})\.us
-    /x
+    /x.freeze
 
     def self.valid?(domain)
       !domain.to_s.match(Locality::REGEX).nil?

data/lib/gman/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class Gman
-  VERSION = '7.0.2'.freeze
+  VERSION = '7.0.3'
 end

data/script/add

@@ -1,4 +1,6 @@
 #! /usr/bin/env ruby
+# frozen_string_literal: true
+
 #
 # Add one or more domains to a given group, running the standard import checks
 #

data/script/alphabetize

@@ -1,4 +1,6 @@
 #! /usr/bin/env ruby
+# frozen_string_literal: true
+
 #
 # Alphabetizes entries in the domains.txt file
 #

data/script/dedupe

@@ -1,4 +1,5 @@
 #! /usr/bin/env ruby
+# frozen_string_literal: true
 
 require 'yaml'
 require 'open-uri'

data/script/profile

@@ -1,4 +1,5 @@
 #! /usr/bin/env ruby
+# frozen_string_literal: true
 
 require 'ruby-prof'
 require './lib/gman'

data/script/prune

@@ -1,4 +1,6 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
+
 # Given an array of domains, removes them from the list
 # Example usage: script/prune foo.invalid, bar.invalid, foo.bar.invalid
 
@@ -12,9 +14,9 @@ list = File.open('./config/domains.txt').read
 puts "Starting list: #{Gman::DomainList.current.count} domains"
 
 domains.each do |domain|
-  list.gsub!(/^#{domain}$\n/, '')
+  list.gsub!(/^#{Regexp.escape(domain)}$\n/, '')
 end
 
-puts "Ending list: #{Gman::DomainList.current.count} domains"
-
 File.write './config/domains.txt', list
+
+puts "Ending list: #{Gman::DomainList.current.count} domains"

data/script/reconcile-us

@@ -1,4 +1,6 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
+
 #
 # Reconciles the USA.gov-maintained list of US domains with domains.txt
 # to show domains listed in the USA.gov-maintained list that we reject and why
@@ -12,7 +14,7 @@ ENV['RECONCILING'] = 'true'
 blacklist = ['usagovQUASI']
 source = 'https://raw.githubusercontent.com/GSA/govt-urls/master/government-urls-hierarchical-list.txt'
 
-data = open(source).read
+data = URI.open(source).read
 data = data.split('_' * 74)
 data = data.last.strip
 data = data.split(/\r?\n/).reject(&:empty?)
@@ -33,7 +35,7 @@ importer = Gman::Importer.new(domains)
 
 importer.logger.info "Starting with #{importer.domains.count} domains"
 
-importer.domains.list.each do |_group, d|
+importer.domains.list.each_value do |d|
   d.map! { |domain| Gman.new(domain).to_s }
   d.map! { |domain| importer.normalize_domain(domain) }
 end
@@ -44,6 +46,7 @@ importer.logger.info "Filtered down to #{count} normalized domains"
 missing = {}
 importer.domains.list.each do |g, usagovdomains|
   next unless importer.current.list[g]
+
   missing[g] = importer.current.list[g] - usagovdomains
 end
 

data/script/vendor-federal-de

@@ -1,4 +1,5 @@
 #! /usr/bin/env ruby
+# frozen_string_literal: true
 
 require 'csv'
 require 'open-uri'
@@ -6,7 +7,7 @@ require './lib/gman'
 
 url = 'https://raw.githubusercontent.com/robbi5/german-gov-domains/master/data/domains.csv'
 
-domains = open(url).read.encode('UTF-8')
+domains = URI.open(url).read.encode('UTF-8')
 domains = CSV.parse(domains, headers: true)
 domains = domains.map { |row| row['Domain Name'] }
 

data/script/vendor-municipal-de

@@ -1,4 +1,5 @@
 #! /usr/bin/env ruby
+# frozen_string_literal: true
 
 require 'csv'
 require 'open-uri'
@@ -6,7 +7,7 @@ require './lib/gman'
 
 url = 'http://www.mik.nrw.de/nc/themen-aufgaben/kommunales/kommunale-adressen.html?tx_szkommunaldb_pi1%5Bexport%5D=csv'
 
-csv = open(url).read.force_encoding('iso-8859-1').encode('UTF-8')
+csv = URI.open(url).read.force_encoding('iso-8859-1').encode('UTF-8')
 
 # For some reason, the header row is actually the last row
 # Pop the last line off the file and prepend it at the begining

data/script/vendor-nl

@@ -1,4 +1,6 @@
 #! /usr/bin/env ruby
+# frozen_string_literal: true
+
 # See https://github.com/github/government.github.com/pull/367#issuecomment-102108763
 
 require 'fileutils'

data/script/vendor-public-suffix

@@ -1,4 +1,6 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
+
 # Propagates an initial list of best-guess government domains
 
 require 'public_suffix'
@@ -6,7 +8,7 @@ require 'yaml'
 require_relative '../lib/gman'
 
 # https://gist.github.com/benbalter/6147066
-REGEX = /(\.g[ou]{1,2}(v|b|vt)|\.mil|\.gc|\.fed)(\.[a-z]{2})?$/i
+REGEX = /(\.g[ou]{1,2}(v|b|vt)|\.mil|\.gc|\.fed)(\.[a-z]{2})?$/i.freeze
 
 domains = []
 PublicSuffix::List.default.each do |rule|

data/script/vendor-se

@@ -1,4 +1,5 @@
 #! /usr/bin/env ruby
+# frozen_string_literal: true
 
 require 'mechanize'
 require 'csv'

data/script/vendor-swot

@@ -1,4 +1,6 @@
 #! /usr/bin/env ruby
+# frozen_string_literal: true
+
 #
 # Vendors the Swot-maintained list of adademic domains into config/academic.txt
 # Source: https://github.com/leereilly/swot/

data/script/vendor-us

@@ -1,4 +1,6 @@
 #! /usr/bin/env ruby
+# frozen_string_literal: true
+
 #
 # Vendors the USA.gov-maintained list of US domains into domains.txt
 # Source: https://github.com/GSA-OCSIT/govt-urls
@@ -13,10 +15,10 @@
 require './lib/gman'
 require 'open-uri'
 
-blacklist = %w(usagovQUASI usagovFEDgov)
+blacklist = %w[usagovQUASI usagovFEDgov]
 source = 'https://raw.githubusercontent.com/GSA/govt-urls/master/government-urls-hierarchical-list.txt'
 
-data = open(source).read
+data = URI.open(source).read
 data = data.split('_' * 74)
 data = data.last.strip
 data = data.split(/\r?\n/).reject(&:empty?)

data/spec/gman/bin_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 RSpec.describe 'Gman bin' do
   let(:domain) { 'whitehouse.gov' }
   let(:args) { [domain] }
@@ -21,7 +23,7 @@ RSpec.describe 'Gman bin' do
     end
 
     it 'knows the type' do
-      expect(output).to match('federal')
+      expect(output).to match(/federal/i)
     end
 
     it 'knows the agency' do
@@ -87,11 +89,11 @@ RSpec.describe 'Gman bin' do
     let(:args) { [txt_path] }
 
     it 'returns only government domains' do
-      expected = <<-EOS
-mr.senator@obama.senate.gov
-president@whitehouse.gov
-commander.in.chief@us.army.mil
-      EOS
+      expected = <<~EXPECTED
+        mr.senator@obama.senate.gov
+        president@whitehouse.gov
+        commander.in.chief@us.army.mil
+      EXPECTED
 
       expect(output).to eql(expected)
     end

data/spec/gman/country_code_spec.rb

@@ -1,10 +1,12 @@
+# frozen_string_literal: true
+
 RSpec.describe 'Gman Country Codes' do
   {
     'whitehouse.gov' => 'United States of America',
-    'foo.gov.uk'     => 'United Kingdom of Great Britain and Northern Ireland',
-    'army.mil'       => 'United States of America',
-    'foo.gc.ca'      => 'Canada',
-    'foo.eu'         => nil
+    'foo.gov.uk' => 'United Kingdom of Great Britain and Northern Ireland',
+    'army.mil' => 'United States of America',
+    'foo.gc.ca' => 'Canada',
+    'foo.eu' => nil
   }.each do |domain, expected_country|
     context "given #{domain.inspect}" do
       subject { Gman.new(domain) }

data/spec/gman/domain_list_spec.rb

@@ -1,8 +1,10 @@
+# frozen_string_literal: true
+
 RSpec.describe Gman::DomainList do
   let(:data) { subject.data }
   let(:canada) { data['Canada municipal'] }
 
-  [:path, :contents, :data].each do |type|
+  %i[path contents data].each do |type|
     context "when initialized by #{type}" do
       subject do
         case type

data/spec/gman/domains_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 RSpec.describe 'Gman domains' do
   let(:resolve_domains?) { ENV['GMAN_RESOLVE_DOMAINS'] == 'true' }
   let(:importer) { Gman::Importer.new({}) }
@@ -12,6 +14,7 @@ RSpec.describe 'Gman domains' do
 
         Parallel.each(domains, in_threads: 4) do |domain|
           next if importer.valid_domain?(domain, options)
+
           invalid_domains.push domain
         end
 

data/spec/gman/identifier_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 RSpec.describe 'Gman identifier' do
   let(:domain) { '' }
   subject { Gman.new(domain) }
@@ -94,6 +96,10 @@ RSpec.describe 'Gman identifier' do
         it 'knows the agency' do
           expect(subject.agency).to eql('Executive Office of the President')
         end
+
+        it 'knows the organization' do
+          expect(subject.organization).to eql('White House')
+        end
       end
 
       context 'a state .gov' do
@@ -161,14 +167,43 @@ RSpec.describe 'Gman identifier' do
           expect(subject.city).to eql('Pittsburgh')
         end
       end
+
+      context 'a city .gov' do
+        let(:domain) { 'ABERDEENMD.GOV' }
+
+        it "knows it's a city" do
+          expect(subject).to be_a_city
+          expect(subject.type).to eql(:city)
+        end
+
+        it 'knows the city' do
+          expect(subject.city).to eql('Aberdeen')
+        end
+
+        it 'knows the state' do
+          expect(subject.state).to eql('MD')
+        end
+
+        it "knows it's a dotgov" do
+          expect(subject).to be_a_dotgov
+        end
+
+        it "know's it's not a state" do
+          expect(subject).to_not be_a_state
+        end
+
+        it "know's it's not a county" do
+          expect(subject).to_not be_a_county
+        end
+      end
     end
   end
 
   context "determining a domain's type" do
     {
-      :unknown            => 'cityofperu.org',
-      :"Canada municipal" => 'acme.ca',
-      :"Canada federal"   => 'canada.ca'
+      unknown: 'cityofperu.org',
+      "Canada municipal": 'acme.ca',
+      "Canada federal": 'canada.ca'
     }.each do |expected, domain|
       context "Given the #{domain} domain" do
         let(:domain) { domain }