gman 4.6.5 → 4.7.0

data/gman.gemspec

@@ -2,7 +2,7 @@ Gem::Specification.new do |s|
   s.name = "gman"
   s.summary = "Check if a given domain or email address belong to a governemnt entity"
   s.description = "A ruby gem to check if the owner of a given email address is working for THE MAN."
-  s.version = '4.6.5'
+  s.version = '4.7.0'
   s.authors = ["Ben Balter"]
   s.email = "ben.balter@github.com"
   s.homepage = "https://github.com/benbalter/gman"

data/lib/gman/domain_list.rb

@@ -0,0 +1,78 @@
+class Gman < NaughtyOrNice
+  class DomainList
+
+    attr_accessor :list
+    alias_method :to_h, :list
+
+    COMMENT_REGEX = /\/\/[\/\s]*(.*)$/i
+
+    def initialize(list)
+      @list = list
+    end
+
+    def groups
+      list.keys
+    end
+
+    def domains
+      list.values.flatten
+    end
+
+    def count
+      domains.count
+    end
+
+    def alphabetize
+      @list = @list.sort_by { |k,v| k.downcase }.to_h
+      @list.each { |group, domains| domains.sort!.uniq! }
+    end
+
+    def write
+      File.write(Gman.list_path, to_public_suffix)
+    end
+
+    def to_public_suffix
+      current_group = ""
+      output = ""
+      list.sort_by { |group, domains| group.downcase }.each do |group, domains|
+        if group != current_group
+          output << "\n\n" unless current_group.empty? # first entry
+          output << "// #{group}\n"
+          current_group = group
+        end
+        output << domains.join("\n")
+      end
+      output
+    end
+
+    def self.current
+      current = File.open(Gman::list_path).read
+      DomainList.from_public_suffix(current)
+    end
+
+    def self.from_public_suffix(string)
+      string = string.gsub(/\r\n?/, "\n").split("\n")
+      hash = array_to_hash(string)
+      DomainList.new(hash)
+    end
+
+    private
+
+    # Given an array of comments/domains in public suffix format
+    # Converts to a hash in the form of :group => [domain1, domain2...]
+    def self.array_to_hash(domains)
+      group = ""
+      domain_hash = {}
+      domains.each do |line|
+        next if line.empty?
+        if match = COMMENT_REGEX.match(line)
+          group = match[1]
+        else
+          domain_hash[group] = [] if domain_hash[group].nil?
+          domain_hash[group].push line.downcase
+        end
+      end
+      domain_hash
+    end
+  end
+end

data/lib/gman/importer.rb

@@ -0,0 +1,135 @@
+# Utility functions for parsing and manipulating public-suffix formatted domain lists
+# Only used in development and not loaded by default
+require 'yaml'
+require 'open-uri'
+require 'net/dns'
+require 'net/dns/resolver'
+require_relative '../gman'
+require_relative './domain_list'
+
+class Gman < NaughtyOrNice
+  class Importer
+
+    attr_accessor :domains
+
+    # Known false positives from vendored lists
+    BLACKLIST = %w[
+      business.centurytel.net
+      chesnee.net
+      citlink.net
+      egovlink.com
+      emainehosting.com
+      fantasyspringsresort.com
+      frontiernet.net
+      hartford-hwp.com
+      homepages.sover.net
+      htc.net
+      koasekabenaki.org
+      kstrom.net
+      laworkforce.net
+      mississippistateparks.reserveamerica.com
+      mylocalgov.com
+      myweb.cebridge.net
+      ncstars.org
+      neagrelations.org
+      qis.net
+      rootsweb.com
+      showcase.netins.net
+      valuworld.com
+      wctc.net
+      webconnections.net
+      webpages.charter.net
+    ]
+
+    def initialize(domains)
+      @domains = DomainList.new(domains)
+    end
+
+    def logger
+      @logger ||= Logger.new(STDOUT)
+    end
+
+    def normalize_domain(domain)
+      domain.to_s.downcase.strip.gsub(/^www./, "").gsub(/\/$/, "")
+    end
+
+    def valid_domain?(domain, options={})
+      return false if domain.empty?
+      return reject(domain, "home. regex")  if domain =~ /^home\./
+      return reject(domain, "user. regex")  if domain =~ /^users?\./
+      return reject(domain, "sites. regex") if domain =~ /^sites?\./
+      return reject(domain, "weebly")       if domain =~ /weebly\.com$/
+      return reject(domain, "govoffice")    if domain =~ /govoffice\d?\.com$/
+      return reject(domain, "homestead")    if domain =~ /homestead\.com$/
+      return reject(domain, "wix.com")      if domain =~ /wix\.com$/
+      return reject(domain, "locality")     if domain =~ Gman::LOCALITY_REGEX
+      return reject(domain, "blacklist")    if BLACKLIST.include?(domain)
+      return reject(domain, "duplicate")    if !options[:skip_dupe] && current.domains.include?(domain)
+      return reject(domain, "invalid")      unless PublicSuffix.valid?(".#{domain}")
+      return reject(domain, "academic")     if Swot::is_academic?(domain)
+      return reject(domain, "unresolvable") if !options[:skip_resolve] && !domain_resolves?(domain)
+      true
+    end
+
+    def reject(domain, reason)
+      logger.info "👎 `#{domain}`: #{reason}"
+      false
+    end
+
+    def current
+      @current ||= DomainList.current
+    end
+
+    def import
+      logger.info "Current: #{Gman::DomainList.current.count} domains"
+      logger.info "Adding: #{domains.count} domains"
+
+      domains.list.each do |group, domains|
+        domains.map!    { |domain| Gman.new(domain).to_s }
+        domains.map!    { |domain| normalize_domain(domain) }
+        domains.select! { |domain| valid_domain?(domain) }
+      end
+
+      logger.info "Filtered to: #{domains.count} domains"
+
+      if domains.count == 0
+        logger.info "Nothing to add. Aborting"
+        exit 0
+      end
+
+      domains.list.each do |group,domains|
+        current.list[group] = [] if current.list[group].nil?
+        current.list[group].concat domains
+        current.list[group].sort! # Alphabetize
+        current.list[group].uniq! # Ensure uniqueness
+      end
+
+      logger.info "New: #{current.count} domains"
+
+      logger.info "Writing to disk..."
+      current.write
+      logger.info "Fin."
+    end
+
+    def resolver
+      @resolver ||= begin
+        resolver = Net::DNS::Resolver.new
+        resolver.nameservers = ["8.8.8.8","8.8.4.4"]
+        resolver
+      end
+    end
+
+    # Verifies that the given domain has an MX record, and thus is valid
+    def domain_resolves?(domain)
+      resolver.search(domain).header.anCount > 0 ||
+      resolver.search(domain, Net::DNS::NS).header.anCount > 0 ||
+      resolver.search(domain, Net::DNS::MX).header.anCount > 0
+    end
+  end
+end
+
+class Gman < NaughtyOrNice
+  def self.import(hash)
+    Gman::Importer.new(hash).import
+  end
+end

data/script/alphabetize

@@ -5,38 +5,8 @@
 # usage: script/alphabetize
 
 require_relative "../lib/gman"
+require_relative "../lib/gman/importer"
 
-# Read in existing list
-domains = File.open(Gman.list_path).read
-domains = domains.gsub /\r\n?/, "\n" #normalize line endings
-domains = domains.split("\n")
-
-# Split list into grouped hash
-group = ""
-domain_hash = {}
-domains.each do |line|
-  next if line.empty?
-  if match = /\/\/[\/\s]*(.*)$/i.match(line)
-    group = match[1]
-  else
-    domain_hash[group] = [] if domain_hash[group].nil?
-    domain_hash[group].push line.downcase
-  end
-end
-
-# Sort by groups
-domain_hash = domain_hash.sort_by { |k,v| k.downcase }.to_h
-
-# Sort within groups
-domain_hash.each do |group, domains|
-  domain_hash[group].sort!
-end
-
-output = ""
-domain_hash.each do |group, domains|
-  output << "// #{group}\n"
-  output << domains.join("\n")
-  output << "\n\n"
-end
-
-File.write Gman.list_path, output.strip
+current = Gman::DomainList.current
+current.alphabetize
+current.write

data/script/cibuild

@@ -4,4 +4,3 @@ set -e
 
 bundle exec rake test
 bundle exec script/dedupe
-bundle exec script/state-domains

data/script/dedupe

@@ -3,36 +3,19 @@
 require 'yaml'
 require 'open-uri'
 require './lib/gman'
-require './lib/gman/parser'
+require './lib/gman/importer'
 
-
-current = Gman::Parser.file_to_array( Gman::list_path )
-domain_hash = Gman::Parser.array_to_hash(current)
-domain_list = domain_hash.flat_map { |k,v| v }
+current = Gman::DomainList.current
 
 puts "Checking for duplicate domains in the domain list..."
-puts "Current list contains #{domain_list.count} domains..."
-
-SOURCE = "https://raw.githubusercontent.com/GSA/govt-urls/master/government-urls.yaml"
-source_hash = YAML.load(open(SOURCE).read)
-source_list = source_hash.flat_map { |k,v| v }
-
-dupes = []
-domain_hash.each do |group,domains|
-  domains.each do |domain|
-    if domain_list.count(domain) > 1 && source_list.count(domain) <= 1
-      dupes.push(domain)
-    end
-  end
-end
+puts "Current list contains #{current.count} domains..."
 
-dupes.uniq!
+dupe = current.count - current.domains.uniq.count
+puts "Found #{dupe} duplicate domains"
+exit 0 if dupe == 0
 
-puts "Found #{dupes.count} dupes!"
+dupes = current.domains.select { |domain| current.domains.count(domain) > 1 }
 
-if dupes.count > 0
-  puts dupes.inspect
-  exit 1
-else
-  exit 0
-end
+puts "Duplicate domains:"
+puts dupes
+exit 1

data/script/prune

@@ -2,16 +2,19 @@
 # Given an array of domains, removes them from the list
 # Example usage: script/prune foo.invalid, bar.invalid, foo.bar.invalid
 
+require_relative "../lib/gman"
+require_relative "../lib/gman/domain_list"
+
 domains = ARGV
 domains = domains.clone.map { |d| d.gsub ",", "" }
 
 list = File.open("./config/domains.txt").read
-puts "Starting list: #{list.size} lines"
+puts "Starting list: #{Gman::DomainList.current.count} domains"
 
 domains.each do |domain|
   list.gsub! /^#{domain}$\n/, ""
 end
 
-puts "Ending list: #{list.size} lines"
+puts "Ending list: #{Gman::DomainList.current.count} domains"
 
 File.write "./config/domains.txt", list

data/script/vendor-federal-de

@@ -0,0 +1,14 @@
+#! /usr/bin/env ruby
+
+require 'csv'
+require 'open-uri'
+require './lib/gman'
+require './lib/gman/importer'
+
+url = "https://raw.githubusercontent.com/robbi5/german-gov-domains/master/data/domains.csv"
+
+domains = open(url).read.force_encoding("iso-8859-1").encode("UTF-8")
+domains = CSV.parse(domains, :headers => true)
+domains = domains.map { |row| row["Domain Name"] }
+
+Gman.import("German Federal" => domains)

data/script/vendor-municipal-de

@@ -0,0 +1,23 @@
+#! /usr/bin/env ruby
+
+require 'csv'
+require 'open-uri'
+require './lib/gman'
+require './lib/gman/importer'
+
+url = "http://www.mik.nrw.de/nc/themen-aufgaben/kommunales/kommunale-adressen.html?tx_szkommunaldb_pi1%5Bexport%5D=csv"
+
+csv = open(url).read.force_encoding("iso-8859-1").encode("UTF-8")
+
+# For some reason, the header row is actually the last row
+# Pop the last line off the file and prepend it at the begining
+# So that when we pass it to CSV it detects the headers properly
+lines = csv.split("\n")
+lines.unshift lines.pop
+csv = lines.join("\n")
+
+# Load municipal domains
+data = CSV.parse(csv, :headers => true, :col_sep => ";")
+domains = data.map { |row| row["Internet"] }
+
+Gman.import("German Municipalities" => domains)

data/script/vendor-nl

@@ -1,5 +1,10 @@
-#!/bin/sh
+#! /usr/bin/env ruby
+# See https://github.com/github/government.github.com/pull/367#issuecomment-102108763
 
-# https://github.com/github/government.github.com/pull/367#issuecomment-102108763
-wget -r -np https://almanak.overheid.nl/
-grep @ -rI almanak.overheid.nl/|cut -f 2 -d @|cut -f 1 -d '"'|grep \\.nl$|sort|uniq
+require 'fileutils'
+
+FileUtils.rm_rf("almanak.overheid.nl")
+domains = `wget -r -np https://almanak.overheid.nl/
+grep @ -rI almanak.overheid.nl/|cut -f 2 -d @|cut -f 1 -d '"'|grep \\.nl$|sort|uniq`
+
+Gman.import("Netherlands" => domains.split("\n"))

data/script/{build → vendor-public-suffix}

@@ -3,14 +3,13 @@
 
 require "public_suffix"
 require "yaml"
+require_relative "../lib/gman"
+require_relative "../lib/gman/importer"
 
 # https://gist.github.com/benbalter/6147066
 REGEX = /(\.g[ou]{1,2}(v|b|vt)|\.mil|\.gc|\.fed)(\.[a-z]{2})?$/i
 
-YAML_FILE = File.dirname(__FILE__) + "/../lib/domains.yml"
-domains = YAML.load_file YAML_FILE
-domains = [] unless domains
-
+domains = []
 PublicSuffix::List.default.each do |rule|
   domain = nil
 
@@ -23,5 +22,4 @@ PublicSuffix::List.default.each do |rule|
   domains.push domain unless domain.nil? or domains.include? domain
 end
 
-domains = domains.sort
-File.open(YAML_FILE, 'w+') {|f| f.write(domains.to_yaml)}
+Gman.import("non-us gov" => domains)

data/script/vendor-se

@@ -2,52 +2,18 @@
 
 require 'mechanize'
 require 'csv'
-require 'swot'
 require './lib/gman'
-require './lib/gman/parser'
+require './lib/gman/importer'
 
 url = "http://www.myndighetsregistret.scb.se/Myndighet.aspx"
 agent = Mechanize.new
 page = agent.get(url)
-form = page.form("form1")
+form = page.forms.first
 form.radiobuttons.find { |r| r.value = "Textfil" }.check
 submit_button = form.buttons.find { |b| b.type == "submit" }
 response = agent.submit(form, submit_button)
 
-domains = []
 rows = CSV.parse(response.content, :headers => true, :col_sep => "\t")
+domains = rows.map { |row| row["Webbadress"] unless row["Namn"] =~ /UNIVERSITET/}
 
-puts "Starting with #{rows.count} domains..."
-
-rows.each do |row|
-  next if Swot.valid?(row["Webbadress"]) # Filter out Swot'd domains
-  next if row["Namn"] =~ /UNIVERSITET/ # Filter out domains that are clearly edu
-  domains.push Gman.new(row["Webbadress"]).domain.to_s.gsub(/^www\./,"")
-end
-
-domains.reject! { |domain| domain.empty? }
-domains.compact!
-domains.uniq!
-domains.select! { |domain| PublicSuffix.valid?(".#{domain}") }
-
-puts "Ended up with #{domains.count} domains."
-
-current = Gman::Parser.file_to_array( Gman::list_path )
-current_hash = Gman::Parser.array_to_hash(current)
-
-current_hash["Swedish Administrative Authorities"] = domains
-current_hash = current_hash.sort_by { |group, domains| group.downcase }
-
-# PublicSuffix Formatted Output
-current_group = ""
-output = ""
-current_hash.each do |group, domains|
-  if group != current_group
-    output << "\n\n" unless current_group.empty? # first entry
-    output << "// #{group}\n"
-    current_group = group
-  end
-  output << domains.join("\n")
-end
-
-File.open(Gman.list_path, "w") { |file| file.write output }
+Gman.import("Swedish Administrative Authorities" => domains)