gman 6.0.1 → 7.0.0

data/script/vendor-municipal-de

@@ -3,7 +3,6 @@
 require 'csv'
 require 'open-uri'
 require './lib/gman'
-require './lib/gman/importer'
 
 url = 'http://www.mik.nrw.de/nc/themen-aufgaben/kommunales/kommunale-adressen.html?tx_szkommunaldb_pi1%5Bexport%5D=csv'
 
@@ -20,4 +19,4 @@ csv = lines.join("\n")
 data = CSV.parse(csv, headers: true, col_sep: ';')
 domains = data.map { |row| row['Internet'] }
 
-Gman.import('German Municipalities' => domains)
+Gman::Importer.new('German Municipalities' => domains).import

data/script/vendor-nl

@@ -2,6 +2,7 @@
 # See https://github.com/github/government.github.com/pull/367#issuecomment-102108763
 
 require 'fileutils'
+require './lib/gman'
 
 FileUtils.rm_rf('almanak.overheid.nl')
 commands = [
@@ -15,4 +16,4 @@ commands = [
 ]
 domains = system commands.join('|')
 
-Gman.import('Netherlands' => domains.split("\n"))
+Gman::Importer.new('Netherlands' => domains.split("\n")).import

data/script/vendor-public-suffix

@@ -4,7 +4,6 @@
 require 'public_suffix'
 require 'yaml'
 require_relative '../lib/gman'
-require_relative '../lib/gman/importer'
 
 # https://gist.github.com/benbalter/6147066
 REGEX = /(\.g[ou]{1,2}(v|b|vt)|\.mil|\.gc|\.fed)(\.[a-z]{2})?$/i
@@ -24,4 +23,4 @@ end
 
 # Note: We want to skip resolution here, because a domain like `gov.sv` may be
 # a valid TLD, not have any top-level sites, and we'd still want it listed
-Gman.import({ 'non-us gov' => domains }, skip_resolve: true)
+Gman::Importer.new('non-us gov' => domains).import(skip_resolve: true)

data/script/vendor-se

@@ -3,7 +3,6 @@
 require 'mechanize'
 require 'csv'
 require './lib/gman'
-require './lib/gman/importer'
 
 url = 'http://www.myndighetsregistret.scb.se/Myndighet.aspx'
 agent = Mechanize.new
@@ -18,4 +17,4 @@ domains = rows.map do |row|
   row['Webbadress'] unless row['Namn'] =~ /UNIVERSITET/
 end
 
-Gman.import('Swedish Administrative Authorities' => domains)
+Gman::Importer.new('Swedish Administrative Authorities' => domains).import

data/script/vendor-swot

@@ -0,0 +1,41 @@
+#! /usr/bin/env ruby
+#
+# Vendors the Swot-maintained list of adademic domains into config/academic.txt
+# Source: https://github.com/leereilly/swot/
+#
+# Usage: script/vendor-swot
+#
+# Will automatically fetch latest version of the list and merge
+# You can check for changes and commit via `git status`
+#
+# It's also probably a good idea to run `script/ci-build` for good measure
+#
+# Note: We do this, because as a bajillion individual files, Swot takes up 30MB
+
+require './lib/gman'
+require 'swot'
+
+# Generate array of all Swot domains
+domains = Swot.all_domains
+domains << Swot::ACADEMIC_TLDS
+
+# Init the importer, builiding a DomainList
+group = "Academic domains vendored from Swot v#{Swot::VERSION}"
+hash = { group => domains }
+
+importer = Gman::Importer.new(hash)
+importer.logger.info "Importing from Swot v#{Swot::VERSION}"
+importer.logger.info "Found #{domains.count} academic domains"
+
+domain_list = importer.domain_list
+domain_list.path = Gman.academic_list_path
+
+# Cleanup and write
+# Note: we're not using the import method, as that assume's we're writing the
+# government domain list and would use Swot to ensure domains aren't academic
+importer.send :normalize_domains!
+domain_list.data[group] << Swot::BLACKLIST.map { |domain| "!#{domain}" }
+domain_list.data[group] = domain_list.data[group].flatten
+domain_list.write
+
+importer.logger.info "Vendored #{importer.domain_list.count} academic domains."

data/script/vendor-us

@@ -10,7 +10,8 @@
 #
 # It's also probably a good idea to run `script/ci-build` for good measure
 
-require './lib/gman/importer'
+require './lib/gman'
+require 'open-uri'
 
 blacklist = %w(usagovQUASI usagovFEDgov)
 source = 'https://raw.githubusercontent.com/GSA/govt-urls/master/government-urls-hierarchical-list.txt'
@@ -32,4 +33,4 @@ data.each do |row|
 end
 
 domains.reject! { |g, _| blacklist.include?(g) }
-Gman.import(domains)
+Gman::Importer.new(domains).import

data/test/helper.rb

@@ -38,3 +38,11 @@ def with_env(key, value)
   yield
   ENV[key] = old_env
 end
+
+def stubbed_list_path
+  File.expand_path './fixtures/domains.txt', File.dirname(__FILE__)
+end
+
+def stubbed_list
+  Gman::DomainList.new(path: stubbed_list_path)
+end

data/test/test_gman.rb

@@ -53,10 +53,4 @@ class TestGman < Minitest::Test
   should 'returns the path to domains.txt' do
     assert_equal true, File.exist?(Gman.list_path)
   end
-
-  should 'stub domains when asked' do
-    with_env 'GMAN_STUB_DOMAINS', 'true' do
-      assert_equal fixture_path('domains.txt'), Gman.list_path
-    end
-  end
 end

data/test/test_gman_domain_list.rb

@@ -0,0 +1,112 @@
+require File.join(File.dirname(__FILE__), 'helper')
+
+class TestGmanDomainList < Minitest::Test
+  INIT_TYPES = [:path, :contents, :data].freeze
+
+  def setup
+    @original_domain_list = File.read(stubbed_list_path)
+  end
+
+  def teardown
+    File.write stubbed_list_path, @original_domain_list
+  end
+
+  def domain_list(type)
+    case type
+    when :path
+      Gman::DomainList.new(path: Gman.list_path)
+    when :contents
+      contents = File.read(Gman.list_path)
+      Gman::DomainList.new(contents: contents)
+    when :data
+      data = Gman::DomainList.new(path: Gman.list_path).to_h
+      Gman::DomainList.new(data: data)
+    end
+  end
+
+  INIT_TYPES.each do |type|
+    context "when initalized with #{type}" do
+      should 'store the init vars' do
+        refute domain_list(type).public_send(type).nil?
+      end
+
+      should 'return the domain data' do
+        list = domain_list(type)
+        assert list.data.key? 'Canada federal'
+        assert list.data.any? { |_key, values| values.include? 'gov' }
+      end
+
+      should 'return the list contents' do
+        list = domain_list(type)
+        assert_match(/^gov$/, list.contents)
+      end
+
+      should 'return the list path' do
+        list = domain_list(type)
+        assert_equal list.path, Gman.list_path
+      end
+
+      should 'return the public suffix parsed list' do
+        list = domain_list(type)
+        assert list.public_suffix_list.class == PublicSuffix::List
+      end
+
+      should 'know if a domain is valid' do
+        list = domain_list(type)
+        assert list.valid? 'whitehouse.gov'
+      end
+
+      should 'know if a domain is invalid' do
+        list = domain_list(type)
+        refute list.valid? 'example.com'
+      end
+
+      should 'return the domain groups' do
+        list = domain_list(type)
+        assert list.groups.include?('Canada federal')
+      end
+
+      should 'return the domains' do
+        list = domain_list(type)
+        assert list.domains.include?('gov')
+      end
+
+      should 'return the domain count' do
+        list = domain_list(type)
+        assert list.count.is_a?(Integer)
+        assert list.count > 100
+      end
+
+      should 'alphabetize the list' do
+        list = domain_list(type)
+        list.data['Canada municipal'].shuffle!
+        assert list.data['Canada municipal'].first != '100milehouse.com'
+        list.alphabetize
+        assert list.data['Canada municipal'].first == '100milehouse.com'
+      end
+
+      should 'write the list' do
+        list = domain_list(type)
+        list.instance_variable_set('@path', stubbed_list_path)
+        list.data = { 'foo' => ['bar.gov', 'baz.net'] }
+        list.write
+        contents = File.read(stubbed_list_path)
+        assert_match %r{^// foo$}, contents
+        expected = "// foo\nbar.gov\nbaz.net"
+        assert contents.include?(expected)
+      end
+
+      should 'output the list in public_suffix format' do
+        list = domain_list(type)
+        string = list.to_s
+        assert_match %r{^// Canada federal$}, string
+        assert string.include? "// Canada federal\ncanada\.ca\n"
+      end
+
+      should "find a domain's parent" do
+        list = domain_list(type)
+        assert_equal 'gov.uk', list.parent_domain('foo.gov.uk')
+      end
+    end
+  end
+end

data/test/test_gman_domains.rb

@@ -20,13 +20,12 @@ class TestGmanDomains < Minitest::Test
     end
 
     invalid = []
-    list = Gman::DomainList.current.list
-    Parallel.each(list, in_threads: 2) do |group, domains|
+    options = { skip_dupe: true, skip_resolve: !resolve_domains? }
+    Gman.list.to_h.each do |group, domains|
       next if WHITELIST.include?(group)
-      invalid.push domains.reject { |domain|
-        options = { skip_dupe: true, skip_resolve: !resolve_domains? }
-        importer.valid_domain?(domain, options)
-      }
+      Parallel.each(domains, in_threads: 4) do |domain|
+        invalid.push(domain) unless importer.valid_domain?(domain, options)
+      end
     end
     assert_equal [], invalid.flatten.reject(&:empty?)
   end

data/test/test_gman_importer.rb

@@ -6,21 +6,17 @@ class TestGManImporter < Minitest::Test
     @stdout = StringIO.new
     @importer.instance_variable_set '@logger', Logger.new(@stdout)
 
-    with_env 'GMAN_STUB_DOMAINS', 'true' do
-      @original_domain_list = File.open(Gman.list_path).read
-    end
+    @original_domain_list = File.read(stubbed_list_path)
   end
 
   def teardown
-    with_env 'GMAN_STUB_DOMAINS', 'true' do
-      File.write Gman.list_path, @original_domain_list
-    end
+    File.write stubbed_list_path, @original_domain_list
   end
 
   should 'init the domain list' do
-    assert_equal Gman::DomainList, @importer.domains.class
-    assert_equal 1, @importer.domains.domains.count
-    assert_equal 'example.com', @importer.domains.domains.first
+    assert_equal Gman::DomainList, @importer.domain_list.class
+    assert_equal 1, @importer.domain_list.count
+    assert_equal 'example.com', @importer.domain_list.domains.first
   end
 
   should 'init the logger' do
@@ -51,43 +47,41 @@ class TestGManImporter < Minitest::Test
     should 'normalize domains within the domain list' do
       importer = Gman::Importer.new 'test' => ['www.EXAMPLE.com/']
       importer.send :normalize_domains!
-      assert_equal 'example.com', importer.domains.domains.first
+      assert_equal 'example.com', importer.domain_list.domains.first
     end
 
     should 'remove invalid domains from the domain list' do
       importer = Gman::Importer.new 'test' => ['foo.github.io', 'example.com']
       importer.instance_variable_set '@logger', Logger.new(@stdout)
 
-      assert_equal 2, importer.domains.domains.count
+      assert_equal 2, importer.domain_list.count
       importer.send :ensure_validity!
-      assert_equal 1, importer.domains.domains.count
+      assert_equal 1, importer.domain_list.count
     end
 
     context 'writing the domain list' do
       should 'add domains to the current domain list' do
-        with_env 'GMAN_STUB_DOMAINS', 'true' do
-          domains = { 'test' => ['example.com'], 'test2' => ['github.com'] }
-          importer = Gman::Importer.new domains
-          importer.send :add_to_current
-          expected = "// test\nexample.com\ngov\n\n// test2\ngithub.com"
-          assert_equal expected, File.open(Gman.list_path).read
-        end
+        domains = { 'test' => ['example.com'], 'test2' => ['github.com'] }
+        importer = Gman::Importer.new domains
+        importer.instance_variable_set '@current', stubbed_list
+        importer.send :add_to_current
+        expected = "// test\nexample.com\ngov\n\n// test2\ngithub.com"
+        assert_equal expected, File.open(stubbed_list_path).read
       end
 
       should 'import' do
-        with_env 'GMAN_STUB_DOMAINS', 'true' do
-          domains = {
-            'test'  => ['www.example.com', 'goo.github.io'],
-            'test2' => ['github.com', 'www.github.com', 'whitehouse.gov']
-          }
-
-          importer = Gman::Importer.new domains
-          importer.instance_variable_set '@logger', Logger.new(@stdout)
-          importer.import(skip_resolve: true)
-
-          expected = "// test\nexample.com\ngov\n\n// test2\ngithub.com"
-          assert_equal expected, File.open(Gman.list_path).read
-        end
+        domains = {
+          'test'  => ['www.example.com', 'goo.github.io'],
+          'test2' => ['github.com', 'www.github.com', 'whitehouse.gov']
+        }
+
+        importer = Gman::Importer.new domains
+        importer.instance_variable_set '@current', stubbed_list
+        importer.instance_variable_set '@logger', Logger.new(@stdout)
+        importer.import(skip_resolve: true)
+
+        expected = "// test\nexample.com\ngov\n\n// test2\ngithub.com"
+        assert_equal expected, File.open(stubbed_list_path).read
       end
     end
   end

metadata

@@ -1,71 +1,71 @@
 --- !ruby/object:Gem::Specification
 name: gman
 version: !ruby/object:Gem::Version
-  version: 6.0.1
+  version: 7.0.0
 platform: ruby
 authors:
 - Ben Balter
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-22 00:00:00.000000000 Z
+date: 2016-03-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: swot
+  name: iso_country_codes
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0.6'
 - !ruby/object:Gem::Dependency
-  name: iso_country_codes
+  name: naughty_or_nice
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: naughty_or_nice
+  name: colored
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '1.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '1.2'
 - !ruby/object:Gem::Dependency
-  name: colored
+  name: swot
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
-  type: :runtime
+        version: '1.0'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -217,6 +217,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".rake_tasks"
 - ".rubocop.yml"
 - ".ruby-version"
 - ".travis.yml"
@@ -228,6 +229,7 @@ files:
 - bin/gman
 - bin/gman_filter
 - config/domains.txt
+- config/vendor/academic.txt
 - config/vendor/dotgovs.csv
 - gman.gemspec
 - lib/gman.rb
@@ -254,6 +256,7 @@ files:
 - script/vendor-nl
 - script/vendor-public-suffix
 - script/vendor-se
+- script/vendor-swot
 - script/vendor-us
 - test/fixtures/domains.txt
 - test/fixtures/obama.txt
@@ -261,6 +264,7 @@ files:
 - test/test_gman.rb
 - test/test_gman_bin.rb
 - test/test_gman_country_codes.rb
+- test/test_gman_domain_list.rb
 - test/test_gman_domains.rb
 - test/test_gman_filter.rb
 - test/test_gman_identifier.rb
@@ -297,6 +301,7 @@ test_files:
 - test/test_gman.rb
 - test/test_gman_bin.rb
 - test/test_gman_country_codes.rb
+- test/test_gman_domain_list.rb
 - test/test_gman_domains.rb
 - test/test_gman_filter.rb
 - test/test_gman_identifier.rb