gman 5.0.6 → 5.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8d98b3b52245844954dec2db49e0016b4c00f620
-  data.tar.gz: d374b390f178b4f2e02e3eea1ccb770891d46070
+  metadata.gz: 5836f10b4bf15f1e7c91b197095ae1dff999bc04
+  data.tar.gz: 49032b8c90816c1fd6a52268f3f212520d07cd99
 SHA512:
-  metadata.gz: c23d968dd4a9bdce198794f4477c645c99642f95cb9134858fed569dea4322a0eb8ec64b08752fd0246389f693d777ede3c6881e5b41c9aa64d7a865d91b3237
-  data.tar.gz: 67c4672fd6bcf4bfbc6705120d31e603903ce44dc2307a0407de644e5712b766ec9fe4ae90f32a3790a51ef4f1ec4efc3c4950ddcd6c0273eb7b343ba31c090e
+  metadata.gz: ba1591c651effddcd8b654316b1ca8a325a705461cb2963cf1a908b1db92e3b2eb3cba0c5ef547103ccc9d7941d2256da5ec720160625b28b91a74a8965c8ff8
+  data.tar.gz: f9dea4ab2d0f97788c479c9f0e8aa5dc9fdda278038c5e201c867d60f6aaf6c2ffefb8c066acae63f1a9bdef535b29424afffbd31a6d2a78047bd50fa862c8ee

data/config/domains.txt

@@ -1201,7 +1201,6 @@ village.hazelton.bc.ca
 village.longview.ab.ca
 village.memramcook.com
 village.merrickville-wolford.on.ca
-village.nobleford.ab.ca
 village.stantoine.nb.ca
 village.westport.on.ca
 villagecharlo.com
@@ -1451,7 +1450,6 @@ riigikogu.ee
 siseministeerium.ee
 sm.ee
 tja.ee
-valitus.ee
 vm.ee
 
 // Finland
@@ -2929,6 +2927,8 @@ gob.mx
 gob.pa
 gob.pe
 gob.pk
+gob.sv
+gob.ve
 gouv.bj
 gouv.ci
 gouv.fr
@@ -3062,6 +3062,7 @@ govt.nz
 gub.uy
 leg.br
 lg.jp
+mil.tr
 nic.in
 onroerenderfgoed.be
 
@@ -4183,6 +4184,7 @@ jeffco.us
 kitcarsoncounty.org
 lakewood.org
 littletongov.org
+metrodenvercfc.org
 metromayors.org
 minturn.org
 mountain-village.co.us
@@ -4192,7 +4194,6 @@ orchardcityco.org
 parachutecolorado.com
 parkco.us
 parkeronline.org
-peakcfc.com
 plattevillegov.org
 prowerscounty.net
 pueblo.org
@@ -4322,6 +4323,7 @@ waterfordct.org
 watertownct.org
 west-hartford.com
 westbrookct.us
+weston-ct.com
 wethersfieldct.com
 willingtonct.org
 wiltonct.org
@@ -5041,7 +5043,6 @@ adeliowa.org
 aftoniowa.com
 akronia.org
 altoona-iowa.com
-anitaiowa.com
 aplingtonia.com
 arnoldsparkcity.com
 baxter-iowa.com
@@ -7759,7 +7760,6 @@ wrightschool.org
 yadkinville.org
 
 // usagovND
-ashley-nd.com
 beulahnd.org
 bismarck.org
 bismarckairport.com
@@ -7839,7 +7839,6 @@ mcville.com
 medorand.com
 mercercountynd.com
 michigannd.com
-milnornd.com
 minnewaukan.com
 minotnd.org
 mohallndak.com
@@ -7869,7 +7868,6 @@ rutlandnd.com
 sargentnd.com
 sheridan.nd.us
 sherwoodnd.com
-sourisnd.com
 stanleynd.com
 stantonnd.com
 steelend.com
@@ -8154,7 +8152,6 @@ brigantinebeachnj.com
 brooklawn-nj.com
 brooklawn.us
 buenaboro.org
-buenavistatownship.org
 burlingtonnj.us
 butlerborough.com
 bwhnj.com
@@ -8736,7 +8733,6 @@ poncatribe-ne.org
 portlions.net
 potawatomi.org
 powhatan.org
-prairieisland.org
 pueblodecochiti.org
 puyallup-tribe.com
 redding-rancheria.com
@@ -8788,7 +8784,6 @@ accessesmeralda.com
 bcnv.org
 carson.org
 churchillcounty.org
-cityoffallon.org
 cityoffernley.org
 cityofhenderson.com
 cityofnorthlasvegas.com
@@ -9175,7 +9170,6 @@ suffernvillage.com
 sylvanbeachny.com
 syrgov.net
 taghkanic.org
-tannersvilleny.org
 tarrytowngov.com
 thurman-ny.com
 tiogacountyny.com
@@ -9396,7 +9390,6 @@ townofwheatland.org
 townofwillsboro.com
 townofwilmington.org
 townofwilton.com
-townofwindham.com
 townofwoodbury.com
 townverona.org
 townwalworthny.com
@@ -9425,7 +9418,6 @@ villageofbarneveld.org
 villageofbath.org
 villageofbergen.com
 villageofbridgewater.org
-villageofbrocton.com
 villageofbronxville.com
 villageofbuchanan.com
 villageofcapevincent.org
@@ -10277,7 +10269,6 @@ paehealth.com
 palmertonborough.com
 palmertwp.com
 paradisetownship.com
-parkercity.org
 parkesburg.org
 parksideboro.com
 patientsafetyauthority.org
@@ -10590,7 +10581,6 @@ greatfallssc.net
 greenvillecounty.org
 hamptoncountysc.org
 hamptonsc.net
-hartsvillesc.com
 horrycounty.org
 iop.net
 jamesislandsc.us
@@ -10622,7 +10612,6 @@ patriotspoint.org
 port-of-charleston.com
 portroyal.org
 prosperitysc.com
-richburgsc.net
 richlandonline.com
 ridgespringsc.com
 santeecooper.com
@@ -10666,12 +10655,10 @@ townofbriarcliffe.us
 townofcampobello.com
 townofedistobeach.com
 townofgraycourt.net
-townofheathsprings.org
 townofhollywood.org
 townofhoneapath.com
 townofirmosc.com
 townofiva.com
-townofjamesislandsc.org
 townofkershaw.net
 townofmcclellanville-sc.net
 townofmccormicksc.org
@@ -11433,7 +11420,6 @@ yesvirginia.org
 
 // usagovVI
 gov.vi
-governordejongh.com
 legvi.org
 nationalarchives.gov.vg
 usviber.org

data/lib/gman/domain_list.rb

@@ -15,7 +15,7 @@ class Gman
     end
 
     def domains
-      list.values.flatten
+      list.values.flatten.sort.uniq
     end
 
     def count

data/lib/gman/importer.rb

@@ -81,7 +81,9 @@ class Gman
       true
     end
 
+    # if RECONCILING=true, return the reason, rather than a bool and silence log output
     def reject(domain, reason)
+      return reason if ENV["RECONCILING"]
       logger.info "👎 `#{domain}`: #{reason}"
       false
     end
@@ -90,14 +92,14 @@ class Gman
       @current ||= DomainList.current
     end
 
-    def import
+    def import(options={})
       logger.info "Current: #{Gman::DomainList.current.count} domains"
       logger.info "Adding: #{domains.count} domains"
 
       domains.list.each do |group, domains|
         domains.map!    { |domain| Gman.new(domain).to_s }
         domains.map!    { |domain| normalize_domain(domain) }
-        domains.select! { |domain| valid_domain?(domain) }
+        domains.select! { |domain| valid_domain?(domain, options) }
       end
 
       logger.info "Filtered to: #{domains.count} domains"
@@ -142,7 +144,7 @@ class Gman
 end
 
 class Gman
-  def self.import(hash)
-    Gman::Importer.new(hash).import
+  def self.import(hash, options={})
+    Gman::Importer.new(hash).import(options)
   end
 end

data/lib/gman/version.rb

@@ -1,3 +1,3 @@
 class Gman
-  VERSION = '5.0.6'
+  VERSION = '5.0.7'
 end

data/script/reconcile-us

@@ -0,0 +1,64 @@
+#!/usr/bin/env ruby
+#
+# Reconciles the USA.gov-maintained list of US domains with domains.txt
+# to show domains listed in the USA.gov-maintained list that we reject and why
+#
+# Usage: script/reconcile-us
+
+require './lib/gman/importer'
+require 'yaml'
+
+ENV["RECONCILING"] = "true"
+blacklist = ["usagovQUASI"]
+source = "https://raw.githubusercontent.com/GSA/govt-urls/master/government-urls-hierarchical-list.txt"
+
+data = open(source).read
+data = data.split("__________________________________________________________________________")
+data = data.last.strip
+data = data.split(/\r?\n/).reject { |r| r.empty? }
+
+domains = {}
+group = ""
+data.each do |row|
+  if row =~ /^\w/
+    group = row
+    domains[group] = []
+  else
+    domains[group].push row.sub("\.\t", "").strip
+  end
+end
+
+domains.reject! { |group,domain| blacklist.include?(group) }
+importer = Gman::Importer.new(domains)
+
+importer.logger.info "Starting with #{importer.domains.count} domains"
+
+importer.domains.list.each do |group, domains|
+  domains.map! { |domain| Gman.new(domain).to_s }
+  domains.map! { |domain| importer.normalize_domain(domain) }
+end
+
+importer.logger.info "Filtered down to #{importer.domains.domains.count} normalized domains"
+
+missing = {}
+importer.domains.list.each do |group, usagovdomains|
+  next unless importer.current.list[group]
+  missing[group] = importer.current.list[group] - usagovdomains
+end
+
+missing.reject! { |key, value| value.empty? }
+
+importer.logger.info "Found #{missing.values.count} domains not on the USA.gov list"
+puts "Here's the list of missing domains:"
+puts YAML.dump(missing)
+
+domains = importer.domains.domains
+domains = domains.group_by { |domain| importer.valid_domain?(domain, :skip_dupe => true) }
+domains.delete(true)
+domains.delete(false)
+domains.delete("locality")
+
+importer.logger.info "Calling out #{domains.values.flatten.count} rejected domains"
+
+puts "Here are the rejected domains and why they were rejected (excluding locality regexs):"
+puts YAML.dump(domains)

data/script/vendor-gov-list

@@ -1,5 +1,17 @@
 #!/bin/sh
+#
+# Vendors the full list of US .gov domains from https://github.com/GSA/data
+# Usage: script/vendor-gov-list
 
-DATE=2015-03-15
+# Set up
+mkdir tmp
+rm -Rf tmp/gsa-data
 
-wget "https://raw.githubusercontent.com/GSA/data/gh-pages/dotgov-domains/$DATE-full.csv" -O config/vendor/dotgovs.csv
+# Vendor the last file in the dotgov-domains folder that ends in `-full.csv`
+git clone https://github.com/GSA/data tmp/gsa-data
+pattern="tmp/gsa-data/dotgov-domains/*-full.csv"
+files=( $pattern )
+cp -f "${files[@]:(-1)}" config/vendor/dotgovs.csv
+
+# Clean up
+rm -Rf tmp/gsa-data

data/script/vendor-public-suffix

@@ -22,4 +22,6 @@ PublicSuffix::List.default.each do |rule|
   domains.push domain unless domain.nil? or domains.include? domain
 end
 
-Gman.import("non-us gov" => domains)
+# Note: We want to skip resolution here, because a domain like `gov.sv` may be
+# a valid TLD, not have any top-level sites, and we'd still want it listed
+Gman.import({"non-us gov" => domains}, :skip_resolve => true)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gman
 version: !ruby/object:Gem::Version
-  version: 5.0.6
+  version: 5.0.7
 platform: ruby
 authors:
 - Ben Balter
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-06 00:00:00.000000000 Z
+date: 2015-10-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: swot
@@ -230,6 +230,7 @@ files:
 - script/dedupe
 - script/profile
 - script/prune
+- script/reconcile-us
 - script/release
 - script/vendor
 - script/vendor-federal-de