swot-ruby 1.0.0

data/data/lib/domains/tlds.txt

@@ -0,0 +1,83 @@
+ac.bd
+ac.be
+ac.gg
+ac.gn
+ac.il
+ac.in
+ac.jp
+ac.kr
+ac.ma
+ac.me
+ac.mw
+ac.ni
+ac.om
+ac.pg
+ac.pr
+ac.ru
+ac.rw
+ac.sz
+ac.yu
+ac.za
+ac.zm
+ed.ao
+ed.cr
+ed.jp
+edu
+edu.al
+edu.ar
+edu.az
+edu.bb
+edu.bd
+edu.bh
+edu.bs
+edu.bz
+edu.ck
+edu.cn
+edu.dz
+edu.ee
+edu.er
+edu.gh
+edu.hn
+edu.in
+edu.jm
+edu.kn
+edu.kz
+edu.lr
+edu.ly
+edu.me
+edu.mg
+edu.ml
+edu.mv
+edu.mw
+edu.ni
+edu.pa
+edu.pr
+edu.pt
+edu.pw
+edu.qa
+edu.sc
+edu.sd
+edu.sh
+edu.sl
+edu.sy
+edu.ws
+edu.ye
+edu.zm
+es.kr
+g12.br
+hs.kr
+ms.kr
+sc.kr
+sch.ae
+sch.gg
+sch.je
+sch.jo
+sch.lk
+sch.ly
+sch.my
+sch.om
+sch.ps
+sch.sa
+school.za
+vic.edu.au
+urfu.me

data/lib/swot/academic_tlds.rb

@@ -0,0 +1,249 @@
+require 'set'
+
+class Swot
+  # These top-level domains are guaranteed to be academic institutions.
+  ACADEMIC_TLDS = %w(
+    ac.ae
+    ac.at
+    ac.bd
+    ac.be
+    ac.cn
+    ac.cr
+    ac.cy
+    ac.fj
+    ac.gg
+    ac.gn
+    ac.id
+    ac.il
+    ac.in
+    ac.ir
+    ac.jp
+    ac.ke
+    ac.kr
+    ac.ma
+    ac.me
+    ac.mu
+    ac.mw
+    ac.mz
+    ac.ni
+    ac.nz
+    ac.om
+    ac.pa
+    ac.pg
+    ac.pr
+    ac.rs
+    ac.ru
+    ac.rw
+    ac.sz
+    ac.th
+    ac.tz
+    ac.ug
+    ac.uk
+    ac.yu
+    ac.za
+    ac.zm
+    ac.zw
+    cc.al.us
+    cc.ar.us
+    cc.az.us
+    cc.ca.us
+    cc.co.us
+    cc.fl.us
+    cc.ga.us
+    cc.hi.us
+    cc.ia.us
+    cc.id.us
+    cc.il.us
+    cc.in.us
+    cc.ks.us
+    cc.ky.us
+    cc.la.us
+    cc.md.us
+    cc.me.us
+    cc.mi.us
+    cc.mn.us
+    cc.mo.us
+    cc.ms.us
+    cc.mt.us
+    cc.nc.us
+    cc.nd.us
+    cc.ne.us
+    cc.nj.us
+    cc.nm.us
+    cc.nv.us
+    cc.ny.us
+    cc.oh.us
+    cc.ok.us
+    cc.or.us
+    cc.pa.us
+    cc.ri.us
+    cc.sc.us
+    cc.sd.us
+    cc.tx.us
+    cc.va.us
+    cc.vi.us
+    cc.wa.us
+    cc.wi.us
+    cc.wv.us
+    cc.wy.us
+    ed.ao
+    ed.cr
+    ed.jp
+    edu
+    edu.af
+    edu.al
+    edu.ar
+    edu.au
+    edu.az
+    edu.ba
+    edu.bb
+    edu.bd
+    edu.bh
+    edu.bi
+    edu.bn
+    edu.bo
+    edu.br
+    edu.bs
+    edu.bt
+    edu.bz
+    edu.ck
+    edu.cn
+    edu.co
+    edu.cu
+    edu.do
+    edu.dz
+    edu.ec
+    edu.ee
+    edu.eg
+    edu.er
+    edu.es
+    edu.et
+    edu.ge
+    edu.gh
+    edu.gr
+    edu.gt
+    edu.hk
+    edu.hn
+    edu.ht
+    edu.in
+    edu.iq
+    edu.jm
+    edu.jo
+    edu.kg
+    edu.kh
+    edu.kn
+    edu.kw
+    edu.ky
+    edu.kz
+    edu.la
+    edu.lb
+    edu.lr
+    edu.lv
+    edu.ly
+    edu.me
+    edu.mg
+    edu.mk
+    edu.ml
+    edu.mm
+    edu.mn
+    edu.mo
+    edu.mt
+    edu.mv
+    edu.mw
+    edu.mx
+    edu.my
+    edu.ni
+    edu.np
+    edu.om
+    edu.pa
+    edu.pe
+    edu.ph
+    edu.pk
+    edu.pl
+    edu.pr
+    edu.ps
+    edu.pt
+    edu.pw
+    edu.py
+    edu.qa
+    edu.rs
+    edu.ru
+    edu.sa
+    edu.sc
+    edu.sd
+    edu.sg
+    edu.sh
+    edu.sl
+    edu.sv
+    edu.sy
+    edu.tr
+    edu.tt
+    edu.tw
+    edu.ua
+    edu.uy
+    edu.ve
+    edu.vn
+    edu.ws
+    edu.ye
+    edu.zm
+    es.kr
+    g12.br
+    hs.kr
+    ms.kr
+    sc.kr
+    sc.ug
+    sch.ae
+    sch.gg
+    sch.id
+    sch.ir
+    sch.je
+    sch.jo
+    sch.lk
+    sch.ly
+    sch.my
+    sch.om
+    sch.ps
+    sch.sa
+    sch.uk
+    school.nz
+    school.za
+    tec.ar.us
+    tec.az.us
+    tec.co.us
+    tec.fl.us
+    tec.ga.us
+    tec.ia.us
+    tec.id.us
+    tec.il.us
+    tec.in.us
+    tec.ks.us
+    tec.ky.us
+    tec.la.us
+    tec.ma.us
+    tec.md.us
+    tec.me.us
+    tec.mi.us
+    tec.mn.us
+    tec.mo.us
+    tec.ms.us
+    tec.mt.us
+    tec.nc.us
+    tec.nd.us
+    tec.nh.us
+    tec.nm.us
+    tec.nv.us
+    tec.ny.us
+    tec.oh.us
+    tec.ok.us
+    tec.pa.us
+    tec.sc.us
+    tec.sd.us
+    tec.tx.us
+    tec.ut.us
+    tec.vi.us
+    tec.wa.us
+    tec.wi.us
+    tec.wv.us
+    vic.edu.au
+  ).to_set.freeze
+end

data/lib/swot/collection_methods.rb

@@ -0,0 +1,20 @@
+# Module for methods that act on the entire Swot dataset.
+
+module SwotCollectionMethods
+
+  # Returns an array of domain strings.
+  def all_domains
+    each_domain.map(&:to_s)
+  end
+
+  # Yields a Swot instance for every domain under lib/domains. Does not
+  #   include blacklisted or ACADEMIC_TLDS domains.
+  #
+  # returns a Enumerator object with Swot instances if no block is given
+  def each_domain
+    return to_enum(:each_domain) unless block_given?
+    Pathname.glob(Pathname.new(Swot.domains_path).join('**/*.txt')) do |path|
+      yield(Swot.from_path(path))
+    end
+  end
+end

data/lib/swot.rb

@@ -0,0 +1,90 @@
+require "public_suffix"
+require "naughty_or_nice"
+require_relative "swot/academic_tlds"
+require_relative "swot/collection_methods"
+
+class Swot
+
+  VERSION = "0.4.2"
+
+  # These are domains that snuck into the edu registry,
+  # but don't pass the education sniff test
+  # Note: validated domain must not end with the blacklisted string
+  BLACKLIST = File.readlines(File.join(__dir__, '../data/lib/domains/stoplist.txt')).map(&:chomp).freeze
+
+  include NaughtyOrNice
+  extend SwotCollectionMethods
+  class << self
+    alias_method :is_academic?, :valid?
+    alias_method :academic?, :valid?
+
+    def get_institution_name(text)
+      Swot.new(text).institution_name
+    end
+    alias_method :school_name, :get_institution_name
+
+    def domains_path
+      @domains_path ||= File.expand_path "../data/lib/domains", File.dirname(__FILE__)
+    end
+
+    # Returns a new Swot instance for the domain file at the given path.
+    #   Note that the path must be absolute.
+    #
+    # Returns a Swot instance or false is no domain is found at the given path.
+    def from_path(path_string_or_path)
+      path = Pathname.new(path_string_or_path)
+      return false unless path.exist?
+      path_dir, file = path.relative_path_from(Pathname.new(domains_path)).split
+      backwards_path = path_dir.to_s.split('/').push(file.basename('.txt').to_s)
+      domain = backwards_path.reverse.join('.')
+      Swot.new(domain)
+    end
+  end
+
+  # Figure out if an email or domain belongs to academic institution.
+  #
+  # Returns true if the domain name belongs to an academic institution;
+  #  false otherwise.
+  def valid?
+    if domain.nil?
+      false
+    elsif BLACKLIST.any? { |d| to_s =~ /(\A|\.)#{Regexp.escape(d)}\z/ }
+      false
+    elsif ACADEMIC_TLDS.include?(domain.tld)
+      true
+    elsif academic_domain?
+      true
+    else
+      false
+    end
+  end
+
+  # Figure out the institution name based on the email address/domain.
+  #
+  # Returns a string with the institution name; nil if nothing is found.
+  def institution_name
+    @institution_name ||= File.read(file_path, :mode => "rb", :external_encoding => "UTF-8").strip
+  rescue
+    nil
+  end
+  alias_method :school_name, :institution_name
+  alias_method :name, :institution_name
+
+  # Figure out if a domain name is a know academic institution.
+  #
+  # Returns true if the domain name belongs to a known academic institution;
+  #  false otherwise.
+  def academic_domain?
+    @academic_domain ||= File.exist?(file_path) || File.exist?(file_extended_path)
+  end
+
+  private
+
+  def file_path
+    @file_path ||= File.join(Swot::domains_path, domain.domain.to_s.split(".").reverse) + ".txt"
+  end
+
+  def file_extended_path
+    @file_extended_path ||= File.join([Swot::domains_path, domain.to_s.split(".").reverse].flatten) + ".txt"
+  end
+end

data/swot.gemspec

@@ -0,0 +1,33 @@
+# -*- encoding: utf-8 -*-
+Gem::Specification.new do |s|
+  s.name = "swot-ruby"
+  s.version = "1.0.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Lee Reilly"]
+  s.date = "2015-07-10"
+  s.description = "Identify email addresses or domains names that belong to colleges or universities. Help automate the process of approving or rejecting academic discounts."
+  s.email = "lee@leereilly.net"
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.md"
+  ]
+
+  s.files = `git ls-files -z`.split("\x0") + Dir['data/lib/domains/*']
+
+  Dir.glob('lib/domains/**/*.txt').each do |path|
+    s.files << path if File.file?(path)
+  end
+
+  s.homepage = "https://github.com/kobaltz/swot"
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.required_ruby_version = Gem::Requirement.new(">= 2.0")
+  s.rubygems_version = "2.0.14"
+  s.summary = "Identify email addresses or domains names that belong to colleges or universities."
+  s.test_files = ["test/helper.rb", "test/test_collection_methods.rb", "test/test_swot.rb"]
+
+  s.add_dependency('public_suffix', ">= 0")
+  s.add_dependency('naughty_or_nice', "~> 2.0")
+end
+

data/test/helper.rb

@@ -0,0 +1,32 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'minitest/autorun'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'swot'
+
+# Extracted from Rails ActiveSupport::Testing::Assertions
+#
+# Assert that an expression is not truthy. Passes if <tt>object</tt> is
+# +nil+ or +false+. "Truthy" means "considered true in a conditional"
+# like <tt>if foo</tt>.
+#
+#   assert_not nil    # => true
+#   assert_not false  # => true
+#   assert_not 'foo'  # => Expected "foo" to be nil or false
+#
+# An error message can be specified.
+#
+#   assert_not foo, 'foo should be false'
+def assert_not(object, message = nil)
+  message ||= "Expected #{mu_pp(object)} to be nil or false"
+  assert !object, message
+end

data/test/test_collection_methods.rb

@@ -0,0 +1,44 @@
+require 'helper'
+require 'fileutils'
+require 'tmpdir'
+
+describe 'SwotCollectionMethods' do
+  before do
+    @tmp_dir = Dir.mktmpdir "swot-test"
+    Swot.instance_variable_set(:@domains_path, @tmp_dir)
+    write_domain_file "students.texas.edu"
+    write_domain_file "mit.edu"
+  end
+
+  after do
+    FileUtils.remove_entry @tmp_dir
+  end
+
+  def write_domain_file(domain, school_name = 'The University')
+    parts = domain.split('.').reverse
+    parts.last.concat('.txt')
+    path = Pathname.new(@tmp_dir).join(*parts)
+    path.dirname.mkpath
+    path.open('w') { |f| f.puts school_name }
+  end
+
+  describe 'all_domains' do
+    it 'gets all domains from files on disk' do
+      all_domains = Swot.all_domains
+      assert_equal all_domains.size, 2
+      assert_includes all_domains, "students.texas.edu"
+      assert_includes all_domains, "mit.edu"
+    end
+  end
+
+  describe 'each_domain' do
+    it 'yields a swot instance for every domain file' do
+      domains = []
+      Swot.each_domain { |d| domains << d }
+      assert_equal domains.size, 2
+      assert_equal true, domains.all?{ |d| d.is_a? Swot }
+      assert_includes domains.map(&:to_s), "students.texas.edu"
+      assert_includes domains.map(&:to_s), "mit.edu"
+    end
+  end
+end

data/test/test_swot.rb

@@ -0,0 +1,122 @@
+# encoding: UTF-8
+require 'helper'
+
+describe Swot do
+  it "recognizes academic email addresses and domains" do
+    assert Swot::is_academic?('lreilly@stanford.edu')
+    assert Swot::is_academic?('LREILLY@STANFORD.EDU')
+    assert Swot::is_academic?('Lreilly@Stanford.Edu')
+    assert Swot::is_academic?('lreilly@slac.stanford.edu')
+    assert Swot::is_academic?('lreilly@strath.ac.uk')
+    assert Swot::is_academic?('lreilly@soft-eng.strath.ac.uk')
+    assert Swot::is_academic?('lee@ugr.es')
+    assert Swot::is_academic?('lee@uottawa.ca')
+    assert Swot::is_academic?('lee@mother.edu.ru')
+    assert Swot::is_academic?('lee@ucy.ac.cy')
+    assert Swot::is_academic?('dave.kimura@osu.edu')
+    assert Swot::is_academic?('kimura.13@osu.edu')
+    assert Swot::is_academic?('dave@daffodil.ac')
+    assert Swot::is_academic?('dave@cti.za.ac')
+    assert Swot::is_academic?('dave@lsst.ac')
+    assert Swot::is_academic?('dave@dcc.netpoint.com.bd')
+    assert Swot::is_academic?('dave@student.gatewayhs.com')
+
+
+    assert_not Swot::is_academic?('support@driftingruby.com')
+    assert_not Swot::is_academic?('lee@leerilly.net')
+    assert_not Swot::is_academic?('lee@gmail.com')
+    assert_not Swot::is_academic?('lee@stanford.edu.com')
+    assert_not Swot::is_academic?('lee@strath.ac.uk.com')
+
+    assert Swot::is_academic?('stanford.edu')
+    assert Swot::is_academic?('slac.stanford.edu')
+    assert Swot::is_academic?('www.stanford.edu')
+    assert Swot::is_academic?('http://www.stanford.edu')
+    assert Swot::is_academic?('http://www.stanford.edu:9393')
+    assert Swot::is_academic?('strath.ac.uk')
+    assert Swot::is_academic?('soft-eng.strath.ac.uk')
+    assert Swot::is_academic?('ugr.es')
+    assert Swot::is_academic?('uottawa.ca')
+    assert Swot::is_academic?('mother.edu.ru')
+    assert Swot::is_academic?('ucy.ac.cy')
+
+    assert_not Swot::is_academic?('leerilly.net')
+    assert_not Swot::is_academic?('gmail.com')
+    assert_not Swot::is_academic?('stanford.edu.com')
+    assert_not Swot::is_academic?('strath.ac.uk.com')
+
+    assert_not Swot::is_academic?(nil)
+    assert_not Swot::is_academic?('')
+    assert_not Swot::is_academic?('the')
+
+    assert Swot::is_academic?(' stanford.edu')
+    assert Swot::is_academic?('lee@strath.ac.uk ')
+    assert_not Swot::is_academic?(' gmail.com ')
+
+    assert Swot::is_academic?('lee@stud.uni-corvinus.hu')
+
+    # overkill
+    assert Swot::is_academic?('lee@harvard.edu')
+    assert Swot::is_academic?('lee@mail.harvard.edu')
+  end
+
+  it "returns name of valid institution" do
+    assert_match "University of Strathclyde",        Swot::get_institution_name('lreilly@cs.strath.ac.uk')
+    assert_match "BRG Fadingerstraße Linz, Austria", Swot::get_institution_name('lreilly@fadi.at')
+  end
+
+  it "returns nil when institution invalid" do
+    assert_not Swot::get_institution_name('foo@shop.com')
+  end
+
+  it "test aliased methods" do
+    assert Swot::academic?('stanford.edu')
+    assert_match "University of Strathclyde", Swot::school_name('lreilly@cs.strath.ac.uk')
+  end
+
+  it "fail blacklisted domains" do
+    ["si.edu", " si.edu ", "imposter@si.edu", "foo.si.edu", "america.edu"].each do |domain|
+      assert_not Swot::is_academic?(domain), "#{domain} should be denied"
+    end
+    ["student.tsu.edu.ph"].each do |domain|
+      assert_not Swot::is_academic?(domain), "#{domain} should be denied"
+    end
+  end
+
+  it "not err on tld-only domains" do
+    Swot::is_academic? ".com"
+    assert_not Swot::is_academic?(".com")
+  end
+
+  it "does not err on invalid domains" do
+    assert_not Swot::is_academic?("foo@bar.invalid")
+  end
+
+  # it "contains only text files" do
+  #   Dir.glob("lib/domains/**/*") do |file|
+  #     if not File.directory?(file)
+  #       assert file.end_with?(".txt"), "#{file} should have a .txt extension"
+  #     end
+  #   end
+  # end
+
+  # it "contains no file with an invalid encoding" do
+  #   Dir.glob("lib/domains/**/*") do |file|
+  #     if not File.directory?(file)
+  #       File.open(file, "r") do |fh|
+  #         assert fh.read.valid_encoding?, "Invalid encoding for #{file}"
+  #       end
+  #     end
+  #   end
+  # end
+
+  # it "contains only file with a single line" do
+  #   Dir.glob("lib/domains/**/*") do |file|
+  #     if not File.directory?(file)
+  #       File.open(file, "r") do |fh|
+  #         assert fh.read.lines.count == 1, "#{file} should only contain one line"
+  #       end
+  #     end
+  #   end
+  # end
+end