gman 7.0.1 → 7.0.2

data/contributing.json

@@ -0,0 +1,32 @@
+{
+  "commit": {
+    "subject_cannot_be_empty": true,
+    "subject_must_be_longer_than": 4,
+    "subject_must_be_shorter_than": 101,
+    "subject_lines_must_be_shorter_than": 51,
+    "subject_must_be_single_line": true,
+    "subject_must_be_in_tense": "imperative",
+    "subject_must_start_with_case": "lower",
+    "subject_must_not_end_with_dot": true,
+    "body_lines_must_be_shorter_than": 73
+  },
+  "pull_request": {
+    "subject_cannot_be_empty": true,
+    "subject_must_be_longer_than": 4,
+    "subject_must_be_shorter_than": 101,
+    "subject_must_be_in_tense": "imperative",
+    "subject_must_start_with_case": "upper",
+    "subject_must_not_end_with_dot": true,
+    "body_cannot_be_empty": true
+  },
+  "issue": {
+    "subject_cannot_be_empty": true,
+    "subject_must_be_longer_than": 4,
+    "subject_must_be_shorter_than": 101,
+    "subject_must_be_in_tense": "imperative",
+    "subject_must_start_with_case": "upper",
+    "subject_must_not_end_with_dot": true,
+    "body_cannot_be_empty": true,
+    "body_must_include_reproduction_steps": true
+  }
+}

data/gman.gemspec

@@ -26,18 +26,16 @@ Gem::Specification.new do |s|
   s.required_ruby_version = '~> 2.0'
 
   s.add_dependency('iso_country_codes', '~> 0.6')
-  s.add_dependency('naughty_or_nice', '~> 2.0')
+  s.add_dependency('naughty_or_nice', '= 2.1')
   s.add_dependency('colored', '~> 1.2')
+  s.add_dependency('public_suffix', '~> 2.0')
 
   s.add_development_dependency('swot', '~> 1.0')
-  s.add_development_dependency('rake', '~> 10.4')
-  s.add_development_dependency('shoulda', '~> 3.5')
-  s.add_development_dependency('rdoc', '~> 4.2')
-  s.add_development_dependency('bundler', '~> 1.10')
   s.add_development_dependency('pry', '~> 0.10')
   s.add_development_dependency('parallel', '~> 1.6')
   s.add_development_dependency('mechanize', '~> 2.7')
   s.add_development_dependency('addressable', '~> 2.3')
   s.add_development_dependency('ruby-prof', '~> 0.15')
   s.add_development_dependency('rubocop', '~> 0.37')
+  s.add_development_dependency('rspec', '~> 3.5')
 end

data/lib/gman.rb

@@ -55,7 +55,7 @@ class Gman
   private
 
   def valid_domain?
-    @valid_domains ||= domain && domain.valid? && !academic?
+    @valid_domains ||= !domain.nil? && !academic?
   end
 
   def academic?

data/lib/gman/domain_list.rb

@@ -58,12 +58,10 @@ class Gman
       @public_suffix_list ||= PublicSuffix::List.parse(contents)
     end
 
-    # domain is on the domain list and
-    # domain is not explicitly blacklisted and
-    # domain matches a standard public suffix list rule
+    # domain is on the domain list
     def valid?(domain)
-      rule = public_suffix_list.find(domain)
-      !rule.nil? && rule.type != :exception && rule.allow?(".#{domain}")
+      rule = public_suffix_list.find(domain, default: nil)
+      !(rule.nil? || rule.is_a?(PublicSuffix::Rule::Exception))
     end
 
     # Returns an array of strings representing the list groups
@@ -82,9 +80,13 @@ class Gman
     end
 
     # Alphabetize groups and domains within each group
+    # We need to ensure exceptions appear after their coresponding rules
     def alphabetize
       @data = data.sort_by { |k, _v| k.downcase }.to_h
-      @data.each { |_group, domains| domains.sort!.uniq! }
+      @data.map do |_group, domains|
+        domains.sort! { |a, b| sort_with_exceptions(a, b) }
+        domains.uniq!
+      end
     end
 
     # Write the domain list to disk
@@ -149,5 +151,15 @@ class Gman
       hash[key] ||= []
       hash[key].push value
     end
+
+    def sort_with_exceptions(a, b)
+      if a.start_with?('!') && !b.start_with?('!')
+        1
+      elsif b.start_with?('!') && !a.start_with?('!')
+        -1
+      else
+        a <=> b
+      end
+    end
   end
 end

data/lib/gman/identifier.rb

@@ -76,7 +76,7 @@ class Gman
     @list_category ||= begin
       match = Gman.list.public_suffix_list.find(domain.to_s)
       return unless match
-      regex = %r{// ([^\n]+)\n?[^/]*\n#{Regexp.escape(match.name)}\n}im
+      regex = %r{// ([^\n]+)\n?[^/]*\n#{Regexp.escape(match.value)}\n}im
       matches = Gman.list.contents.match(regex)
       matches[1] if matches
     end
@@ -91,7 +91,7 @@ class Gman
     return @dotgov_listing if defined? @dotgov_listing
     return unless dotgov?
     @dotgov_listing = Gman.dotgov_list.find do |listing|
-      listing['Domain Name'].casecmp("#{domain.sld}.gov") == 0
+      listing['Domain Name'].casecmp("#{domain.sld}.gov").zero?
     end
   end
 

data/lib/gman/importer.rb

@@ -125,7 +125,7 @@ class Gman
       return false if domain.empty?
       if BLACKLIST.include?(domain)
         reject(domain, 'blacklist')
-      elsif !PublicSuffix.valid?(".#{domain}")
+      elsif !PublicSuffix.valid?("foo.#{domain}")
         reject(domain, 'invalid')
       elsif Swot.is_academic?(domain)
         reject(domain, 'academic')

data/lib/gman/version.rb

@@ -1,3 +1,3 @@
 class Gman
-  VERSION = '7.0.1'.freeze
+  VERSION = '7.0.2'.freeze
 end

data/script/cibuild

@@ -2,7 +2,7 @@
 
 set -ex
 
-bundle exec rake test
+bundle exec rspec
 bundle exec rubocop -D -S -a
 bundle exec script/dedupe
 bundle exec gem build gman.gemspec

data/script/dedupe

@@ -12,7 +12,7 @@ puts "Current list contains #{current.count} domains..."
 
 dupe = current.count - current.domains.uniq.count
 puts "Found #{dupe} duplicate domains"
-exit 0 if dupe == 0
+exit 0 if dupe.zero?
 
 dupes = current.domains.select { |domain| current.domains.count(domain) > 1 }
 

data/script/vendor-swot

@@ -12,7 +12,7 @@
 #
 # Note: We do this, because as a bajillion individual files, Swot takes up 30MB
 
-require './lib/gman'
+require 'gman'
 require 'swot'
 
 # Generate array of all Swot domains

data/spec/fixtures/domains.txt

@@ -0,0 +1,4 @@
+// foo
+bar.gov
+baz.net
+!mail.bar.gov

data/spec/gman/bin_spec.rb

@@ -0,0 +1,99 @@
+RSpec.describe 'Gman bin' do
+  let(:domain) { 'whitehouse.gov' }
+  let(:args) { [domain] }
+  let(:command) { 'gman' }
+  let(:bin_path) do
+    File.expand_path "../../bin/#{command}", File.dirname(__FILE__)
+  end
+  let(:response_parts) { Open3.capture2e('bundle', 'exec', bin_path, *args) }
+  let(:output) { response_parts[0] }
+  let(:status) { response_parts[1] }
+  let(:exit_code) { status.exitstatus }
+
+  context 'a valid domain' do
+    it 'parses the domain' do
+      expect(output).to match('Domain  : whitehouse.gov')
+    end
+
+    it "knows it's valid" do
+      expect(output).to match('Valid government domain')
+      expect(exit_code).to eql(0)
+    end
+
+    it 'knows the type' do
+      expect(output).to match('federal')
+    end
+
+    it 'knows the agency' do
+      expect(output).to match('Executive Office of the President')
+    end
+
+    it 'knows the country' do
+      expect(output).to match('United States')
+    end
+
+    it 'knows the city' do
+      expect(output).to match('Washington')
+    end
+
+    it 'knows the state' do
+      expect(output).to match('DC')
+    end
+
+    it 'colors by default' do
+      expect(output).to match(/\e\[32m/)
+    end
+
+    context 'with colorization disabled' do
+      let(:args) { [domain, '--no-color'] }
+
+      it "doesn't color" do
+        expect(output).to_not match(/\e\[32m/)
+      end
+    end
+  end
+
+  context 'with no args' do
+    let(:args) { [] }
+
+    it 'displays the help text' do
+      expect(output).to match('USAGE')
+    end
+  end
+
+  context 'an invalid domain' do
+    let(:domain) { 'foo.invalid' }
+
+    it 'knows the domain is invalid' do
+      expect(output).to match('Invalid domain')
+      expect(exit_code).to eql(1)
+    end
+  end
+
+  context 'a non-government domain' do
+    let(:domain) { 'github.com' }
+
+    it "knows it's not a government domain" do
+      expect(output).to match('Not a government domain')
+      expect(exit_code).to eql(1)
+    end
+  end
+
+  context 'filtering' do
+    let(:command) { 'gman_filter' }
+    let(:txt_path) do
+      File.expand_path '../fixtures/obama.txt', File.dirname(__FILE__)
+    end
+    let(:args) { [txt_path] }
+
+    it 'returns only government domains' do
+      expected = <<-EOS
+mr.senator@obama.senate.gov
+president@whitehouse.gov
+commander.in.chief@us.army.mil
+      EOS
+
+      expect(output).to eql(expected)
+    end
+  end
+end

data/spec/gman/country_code_spec.rb

@@ -0,0 +1,36 @@
+RSpec.describe 'Gman Country Codes' do
+  {
+    'whitehouse.gov' => 'United States of America',
+    'foo.gov.uk'     => 'United Kingdom of Great Britain and Northern Ireland',
+    'army.mil'       => 'United States of America',
+    'foo.gc.ca'      => 'Canada',
+    'foo.eu'         => nil
+  }.each do |domain, expected_country|
+    context "given #{domain.inspect}" do
+      subject { Gman.new(domain) }
+      let(:country) { subject.country }
+
+      it 'knows the country' do
+        if expected_country.nil?
+          expect(country).to be_nil
+        else
+          expect(country.name).to eql(expected_country)
+        end
+      end
+
+      it 'knows the alpha2' do
+        expected = case expected_country
+                   when 'United States of America'
+                     'us'
+                   when 'Canada'
+                     'ca'
+                   when 'United Kingdom of Great Britain and Northern Ireland'
+                     'gb'
+                   else
+                     'eu'
+                   end
+        expect(subject.alpha2).to eql(expected)
+      end
+    end
+  end
+end

data/spec/gman/domain_list_spec.rb

@@ -0,0 +1,108 @@
+RSpec.describe Gman::DomainList do
+  let(:data) { subject.data }
+  let(:canada) { data['Canada municipal'] }
+
+  [:path, :contents, :data].each do |type|
+    context "when initialized by #{type}" do
+      subject do
+        case type
+        when :path
+          described_class.new(path: Gman.list_path)
+        when :contents
+          contents = File.read(Gman.list_path)
+          described_class.new(contents: contents)
+        when :data
+          data = described_class.new(path: Gman.list_path).to_h
+          described_class.new(data: data)
+        end
+      end
+
+      it 'stores the init var' do
+        expect(subject.send(type)).to_not be_nil
+      end
+
+      it 'returns the domain data' do
+        expect(data).to have_key('Canada federal')
+        expect(data.values.flatten).to include('gov')
+      end
+
+      it 'returns the list contents' do
+        expect(subject.contents).to match(/^gov$/)
+      end
+
+      it 'knows the list path' do
+        expect(subject.path).to eql(Gman.list_path)
+      end
+
+      it 'returns the PublicSuffix list' do
+        expect(subject.public_suffix_list).to be_a(PublicSuffix::List)
+      end
+
+      it 'knows if a domain is valid' do
+        expect(subject.valid?('whitehouse.gov')).to eql(true)
+      end
+
+      it 'knows if a domain is invalid' do
+        expect(subject.valid?('example.com')).to eql(false)
+      end
+
+      it 'returns the domain groups' do
+        expect(subject.groups).to include('Canada federal')
+      end
+
+      it 'returns the domains' do
+        expect(subject.domains).to include('gov')
+      end
+
+      it 'returns the domain count' do
+        expect(subject.count).to be_a(Integer)
+        expect(subject.count).to be > 100
+      end
+
+      it 'alphabetizes the list' do
+        canada.shuffle!
+        expect(canada.first).to_not eql('100milehouse.com')
+        subject.alphabetize
+        expect(canada.first).to eql('100milehouse.com')
+      end
+
+      it 'outputs public suffix format' do
+        expect(subject.to_s).to match("// Canada federal\ncanada\.ca\n")
+      end
+
+      it "finds a domain's parent" do
+        expect(subject.parent_domain('foo.gov.uk')).to eql('gov.uk')
+      end
+
+      context 'with the list path stubbed' do
+        let(:stubbed_file_contents) { File.read(stubbed_list_path) }
+
+        before do
+          subject.instance_variable_set('@path', stubbed_list_path)
+        end
+
+        context 'with list data stubbed' do
+          before do
+            subject.data = { 'foo' => ['!mail.bar.gov', 'bar.gov', 'baz.net'] }
+          end
+
+          context 'alphabetizing' do
+            before { subject.alphabetize }
+
+            it 'puts exceptions last' do
+              expect(subject.data['foo'].last).to eql('!mail.bar.gov')
+            end
+          end
+
+          context 'writing' do
+            before { subject.write }
+
+            it 'writes the contents' do
+              expect(stubbed_file_contents).to match("// foo\nbar.gov\nbaz.net")
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/spec/gman/domains_spec.rb

@@ -0,0 +1,22 @@
+RSpec.describe 'Gman domains' do
+  let(:resolve_domains?) { ENV['GMAN_RESOLVE_DOMAINS'] == 'true' }
+  let(:importer) { Gman::Importer.new({}) }
+  let(:options) { { skip_dupe: true, skip_resolve: !resolve_domains? } }
+
+  Gman.list.to_h.each do |group, domains|
+    next if ['non-us gov', 'non-us mil', 'US Federal'].include?(group)
+
+    context "the #{group} group" do
+      it 'only contains valid domains' do
+        invalid_domains = []
+
+        Parallel.each(domains, in_threads: 4) do |domain|
+          next if importer.valid_domain?(domain, options)
+          invalid_domains.push domain
+        end
+
+        expect(invalid_domains).to be_empty
+      end
+    end
+  end
+end