public_suffix 1.5.3 → 2.0.0

data/lib/public_suffix/version.rb

@@ -3,20 +3,10 @@
 #
 # Domain name parser based on the Public Suffix List.
 #
-# Copyright (c) 2009-2015 Simone Carletti <weppos@weppos.net>
+# Copyright (c) 2009-2016 Simone Carletti <weppos@weppos.net>
 #
 
 module PublicSuffix
-
-  module Version
-    MAJOR = 1
-    MINOR = 5
-    PATCH = 3
-    BUILD = nil
-
-    STRING = [MAJOR, MINOR, PATCH, BUILD].compact.join(".")
-  end
-
-  VERSION = Version::STRING
-
+  # The current library version.
+  VERSION = "2.0.0".freeze
 end

data/public_suffix.gemspec

@@ -1,17 +1,17 @@
 # -*- encoding: utf-8 -*-
-# stub: public_suffix 1.5.2 ruby lib
+# stub: public_suffix 1.5.3 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "public_suffix"
-  s.version = "1.5.2"
+  s.version = "1.5.3"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Simone Carletti"]
-  s.date = "2015-10-27"
+  s.date = "2015-12-14"
   s.description = "PublicSuffix can parse and decompose a domain name into top level domain, domain and subdomains."
   s.email = "weppos@weppos.net"
-  s.files = [".gemtest", ".gitignore", ".ruby-gemset", ".travis.yml", ".yardopts", "CHANGELOG.md", "Gemfile", "LICENSE.txt", "README.md", "Rakefile", "data/definitions.txt", "lib/public_suffix.rb", "lib/public_suffix/domain.rb", "lib/public_suffix/errors.rb", "lib/public_suffix/list.rb", "lib/public_suffix/rule.rb", "lib/public_suffix/version.rb", "public_suffix.gemspec", "test/acceptance_test.rb", "test/test_helper.rb", "test/unit/domain_test.rb", "test/unit/errors_test.rb", "test/unit/list_test.rb", "test/unit/public_suffix_test.rb", "test/unit/rule_test.rb"]
+  s.files = [".gitignore", ".ruby-gemset", ".travis.yml", ".yardopts", "CHANGELOG.md", "Gemfile", "LICENSE.txt", "README.md", "Rakefile", "data/definitions.txt", "lib/public_suffix.rb", "lib/public_suffix/domain.rb", "lib/public_suffix/errors.rb", "lib/public_suffix/list.rb", "lib/public_suffix/rule.rb", "lib/public_suffix/version.rb", "public_suffix.gemspec", "test/acceptance_test.rb", "test/benchmark_helper.rb", "test/execution_profiler.rb", "test/initialization_profiler.rb", "test/performance_benchmark.rb", "test/psl_test.rb", "test/test_helper.rb", "test/tests.txt", "test/unit/domain_test.rb", "test/unit/errors_test.rb", "test/unit/list_test.rb", "test/unit/public_suffix_test.rb", "test/unit/rule_test.rb"]
   s.homepage = "http://simonecarletti.com/code/publicsuffix"
   s.licenses = ["MIT"]
   s.required_ruby_version = Gem::Requirement.new(">= 2.0")

data/test/acceptance_test.rb

@@ -1,46 +1,48 @@
-require 'test_helper'
+require "test_helper"
 
 class AcceptanceTest < Minitest::Unit::TestCase
 
-  ValidCases = [
-      ["google.com",              [nil, "google", "com"]],
-      ["foo.google.com",          ["foo", "google", "com"]],
+  VALID_CASES = [
+      ["example.com",             "example.com",        [nil, "example", "com"]],
+      ["foo.example.com",         "example.com",        ["foo", "example", "com"]],
 
-      ["verybritish.co.uk",       [nil, "verybritish", "co.uk"]],
-      ["foo.verybritish.co.uk",   ["foo", "verybritish", "co.uk"]],
+      ["verybritish.co.uk",       "verybritish.co.uk",  [nil, "verybritish", "co.uk"]],
+      ["foo.verybritish.co.uk",   "verybritish.co.uk",  ["foo", "verybritish", "co.uk"]],
 
-      ["parliament.uk",           [nil, "parliament", "uk"]],
-      ["foo.parliament.uk",       ["foo", "parliament", "uk"]],
-  ]
+      ["parliament.uk",           "parliament.uk",      [nil, "parliament", "uk"]],
+      ["foo.parliament.uk",       "parliament.uk",      ["foo", "parliament", "uk"]],
+  ].freeze
 
   def test_valid
-    ValidCases.each do |name, results|
-      domain = PublicSuffix.parse(name)
+    VALID_CASES.each do |input, domain, results|
+      parsed = PublicSuffix.parse(input)
       trd, sld, tld = results
-      assert_equal tld, domain.tld, "Invalid tld for '#{name}'"
-      assert_equal sld, domain.sld, "Invalid sld for '#{name}'"
-      assert_equal trd, domain.trd, "Invalid trd for '#{name}'"
-      assert PublicSuffix.valid?(name)
+      assert_equal tld, parsed.tld, "Invalid tld for `#{name}`"
+      assert_equal sld, parsed.sld, "Invalid sld for `#{name}`"
+      assert_equal trd, parsed.trd, "Invalid trd for `#{name}`"
+
+      assert_equal domain, PublicSuffix.domain(input)
+      assert PublicSuffix.valid?(input)
     end
   end
 
 
-  InvalidCases = [
+  INVALID_CASES = [
       ["nic.ke",                  PublicSuffix::DomainNotAllowed],
       [nil,                       PublicSuffix::DomainInvalid],
       ["",                        PublicSuffix::DomainInvalid],
       ["  ",                      PublicSuffix::DomainInvalid],
-  ]
+  ].freeze
 
   def test_invalid
-    InvalidCases.each do |(name, error)|
+    INVALID_CASES.each do |(name, error)|
       assert_raises(error) { PublicSuffix.parse(name) }
       assert !PublicSuffix.valid?(name)
     end
   end
 
 
-  RejectedCases = [
+  REJECTED_CASES = [
       ["www. .com",           true],
       ["foo.co..uk",          true],
       ["goo,gle.com",         true],
@@ -48,27 +50,29 @@ class AcceptanceTest < Minitest::Unit::TestCase
       ["google-.com",         true],
 
       # This case was covered in GH-15.
-      # I decide to cover this case because it's not easily reproducible with URI.parse
+      # I decided to cover this case because it's not easily reproducible with URI.parse
       # and can lead to several false positives.
       ["http://google.com",   false],
-  ]
+  ].freeze
 
   def test_rejected
-    RejectedCases.each do |name, expected|
-      assert_equal expected, PublicSuffix.valid?(name)
-      assert !valid_domain?(name), "#{name} expected to be invalid"
+    REJECTED_CASES.each do |name, expected|
+      assert_equal expected, PublicSuffix.valid?(name),
+                   "Expected %s to be %s" % [name.inspect, expected.inspect]
+      assert !valid_domain?(name),
+             "#{name} expected to be invalid"
     end
   end
 
-  
-  CaseCases = [
-      ["Www.google.com",          ["www", "google", "com"]],
-      ["www.Google.com",          ["www", "google", "com"]],
-      ["www.google.Com",          ["www", "google", "com"]],
-  ]
+
+  CASE_CASES = [
+      ["Www.google.com", %w( www google com )],
+      ["www.Google.com", %w( www google com )],
+      ["www.google.Com", %w( www google com )],
+  ].freeze
 
   def test_ignore_case
-    CaseCases.each do |name, results|
+    CASE_CASES.each do |name, results|
       domain = PublicSuffix.parse(name)
       trd, sld, tld = results
       assert_equal tld, domain.tld, "Invalid tld for `#{name}'"
@@ -79,18 +83,37 @@ class AcceptanceTest < Minitest::Unit::TestCase
   end
 
 
+  INCLUDE_PRIVATE_CASES = [
+      ["blogspot.com", true, "blogspot.com"],
+      ["blogspot.com", false, nil],
+      ["subdomain.blogspot.com", true, "blogspot.com"],
+      ["subdomain.blogspot.com", false, "subdomain.blogspot.com"],
+  ].freeze
+
+  def test_ignore_private
+    # test domain and parse
+    INCLUDE_PRIVATE_CASES.each do |given, ignore_private, expected|
+      assert_equal expected, PublicSuffix.domain(given, ignore_private: ignore_private)
+    end
+    # test valid?
+    INCLUDE_PRIVATE_CASES.each do |given, ignore_private, expected|
+      assert_equal !expected.nil?, PublicSuffix.valid?(given, ignore_private: ignore_private)
+    end
+  end
+
+
   def valid_uri?(name)
     uri = URI.parse(name)
-    uri.host != nil
+    !uri.host.nil?
   rescue
     false
   end
 
   def valid_domain?(name)
     uri = URI.parse(name)
-    uri.host != nil && uri.scheme.nil?
+    !uri.host.nil? && uri.scheme.nil?
   rescue
     false
   end
 
-end
+end

data/test/benchmark_helper.rb

@@ -0,0 +1,4 @@
+require "benchmark"
+
+$LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
+require "public_suffix"

data/test/execution_profiler.rb

@@ -0,0 +1,14 @@
+$LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
+
+require "memory_profiler"
+require "public_suffix"
+
+PublicSuffix::List.default
+
+report = MemoryProfiler.report do
+  PublicSuffix.domain("www.example.com")
+  PublicSuffix.domain("a.b.ide.kyoto.jp")
+end
+
+report.pretty_print
+# report.pretty_print(to_file: 'profiler-%s-%d.txt' % [ARGV[0], Time.now.to_i])

data/test/initialization_profiler.rb

@@ -0,0 +1,11 @@
+$LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
+
+require "memory_profiler"
+require "public_suffix"
+
+report = MemoryProfiler.report do
+  PublicSuffix::List.default
+end
+
+report.pretty_print
+# report.pretty_print(to_file: 'profiler-%s-%d.txt' % [ARGV[0], Time.now.to_i])

data/test/performance_benchmark.rb

@@ -0,0 +1,38 @@
+require_relative "benchmark_helper"
+
+iterations = 100_000
+
+# force load
+list = PublicSuffix::List.default
+
+Benchmark.bmbm do |bm|
+  bm.report "Top level TLD" do
+    iterations.times do
+      PublicSuffix.domain("example.com", list)
+    end
+  end
+
+  bm.report "Top level TLD (subdomain)" do
+    iterations.times do
+      PublicSuffix.domain("www.example.com", list)
+    end
+  end
+
+  bm.report "Unlisted TLD" do
+    iterations.times do
+      PublicSuffix.domain("example.example", list)
+    end
+  end
+
+  bm.report "Unlisted TLD (subdomain)" do
+    iterations.times do
+      PublicSuffix.domain("www.example.example", list)
+    end
+  end
+
+  bm.report "Crazy suffix" do
+    iterations.times do
+      PublicSuffix.domain("a.b.ide.kyoto.jp", list)
+    end
+  end
+end

data/test/psl_test.rb

@@ -0,0 +1,49 @@
+require "test_helper"
+require "public_suffix"
+
+# This test runs against the current PSL file and ensures
+# the definitions satisfies the test suite.
+class PslTest < Minitest::Unit::TestCase
+
+  ROOT = File.expand_path("../../", __FILE__)
+
+  # rubocop:disable Lint/Eval
+  def self.tests
+    File.readlines(File.join(ROOT, "test/tests.txt")).map do |line|
+      line = line.strip
+      next if line.empty?
+      next if line.start_with?("//")
+      input, output = line.split(", ")
+
+      # handle the case of eval("null"), it must be eval("nil")
+      input  = "nil" if input  == "null"
+      output = "nil" if output == "null"
+
+      input  = eval(input)
+      output = eval(output)
+      [input, output]
+    end
+  end
+  # rubocop:enable
+
+
+  def test_valid
+    # Parse the PSL and run the tests
+    data = File.read(PublicSuffix::List::DEFAULT_LIST_PATH)
+    PublicSuffix::List.default = PublicSuffix::List.parse(data)
+
+    failures = []
+    self.class.tests.each do |input, output|
+      # Punycode domains are not supported ATM
+      next if input =~ /xn\-\-/
+
+      domain = PublicSuffix.domain(input) rescue nil
+      failures << [input, output, domain] if output != domain
+    end
+
+    message = "The following #{failures.size} tests fail:\n"
+    failures.each { |i, o, d| message += "Expected %s to be %s, got %s\n" % [i.inspect, o.inspect, d.inspect] }
+    assert_equal 0, failures.size, message
+  end
+
+end

data/test/test_helper.rb

@@ -1,9 +1,16 @@
-require 'rubygems'
-require 'minitest/autorun'
-require 'mocha/setup'
+if ENV["COVERALL"]
+  require "coveralls"
+  Coveralls.wear!
+end
+
+require "minitest/autorun"
+require "minitest/reporters"
+require "mocha/setup"
+
+Minitest::Reporters.use! Minitest::Reporters::DefaultReporter.new(color: true)
 
-$:.unshift File.expand_path('../../lib', __FILE__)
-require 'public_suffix'
+$LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
+require "public_suffix"
 
 Minitest::Unit::TestCase.class_eval do
   def assert_not_equal(exp, act, msg = nil)

data/test/tests.txt

@@ -0,0 +1,98 @@
+// Any copyright is dedicated to the Public Domain.
+// http://creativecommons.org/publicdomain/zero/1.0/
+
+// null input
+null, null
+// Mixed case
+'COM', null
+'example.COM', 'example.com'
+'WwW.example.COM', 'example.com'
+// Leading dot
+'.com', null
+'.example', null
+'.example.com', null
+'.example.example', null
+// Unlisted TLD
+'example', null
+'example.example', 'example.example'
+'b.example.example', 'example.example'
+'a.b.example.example', 'example.example'
+// Listed, but non-Internet, TLD
+//'local', null
+//'example.local', null
+//'b.example.local', null
+//'a.b.example.local', null
+// TLD with only 1 rule
+'biz', null
+'domain.biz', 'domain.biz'
+'b.domain.biz', 'domain.biz'
+'a.b.domain.biz', 'domain.biz'
+// TLD with some 2-level rules
+'com', null
+'example.com', 'example.com'
+'b.example.com', 'example.com'
+'a.b.example.com', 'example.com'
+'uk.com', null
+'example.uk.com', 'example.uk.com'
+'b.example.uk.com', 'example.uk.com'
+'a.b.example.uk.com', 'example.uk.com'
+'test.ac', 'test.ac'
+// TLD with only 1 (wildcard) rule
+'mm', null
+'c.mm', null
+'b.c.mm', 'b.c.mm'
+'a.b.c.mm', 'b.c.mm'
+// More complex TLD
+'jp', null
+'test.jp', 'test.jp'
+'www.test.jp', 'test.jp'
+'ac.jp', null
+'test.ac.jp', 'test.ac.jp'
+'www.test.ac.jp', 'test.ac.jp'
+'kyoto.jp', null
+'test.kyoto.jp', 'test.kyoto.jp'
+'ide.kyoto.jp', null
+'b.ide.kyoto.jp', 'b.ide.kyoto.jp'
+'a.b.ide.kyoto.jp', 'b.ide.kyoto.jp'
+'c.kobe.jp', null
+'b.c.kobe.jp', 'b.c.kobe.jp'
+'a.b.c.kobe.jp', 'b.c.kobe.jp'
+'city.kobe.jp', 'city.kobe.jp'
+'www.city.kobe.jp', 'city.kobe.jp'
+// TLD with a wildcard rule and exceptions
+'ck', null
+'test.ck', null
+'b.test.ck', 'b.test.ck'
+'a.b.test.ck', 'b.test.ck'
+'www.ck', 'www.ck'
+'www.www.ck', 'www.ck'
+// US K12
+'us', null
+'test.us', 'test.us'
+'www.test.us', 'test.us'
+'ak.us', null
+'test.ak.us', 'test.ak.us'
+'www.test.ak.us', 'test.ak.us'
+'k12.ak.us', null
+'test.k12.ak.us', 'test.k12.ak.us'
+'www.test.k12.ak.us', 'test.k12.ak.us'
+// IDN labels
+'食狮.com.cn', '食狮.com.cn'
+'食狮.公司.cn', '食狮.公司.cn'
+'www.食狮.公司.cn', '食狮.公司.cn'
+'shishi.公司.cn', 'shishi.公司.cn'
+'公司.cn', null
+'食狮.中国', '食狮.中国'
+'www.食狮.中国', '食狮.中国'
+'shishi.中国', 'shishi.中国'
+'中国', null
+// Same as above, but punycoded
+'xn--85x722f.com.cn', 'xn--85x722f.com.cn'
+'xn--85x722f.xn--55qx5d.cn', 'xn--85x722f.xn--55qx5d.cn'
+'www.xn--85x722f.xn--55qx5d.cn', 'xn--85x722f.xn--55qx5d.cn'
+'shishi.xn--55qx5d.cn', 'shishi.xn--55qx5d.cn'
+'xn--55qx5d.cn', null
+'xn--85x722f.xn--fiqs8s', 'xn--85x722f.xn--fiqs8s'
+'www.xn--85x722f.xn--fiqs8s', 'xn--85x722f.xn--fiqs8s'
+'shishi.xn--fiqs8s', 'shishi.xn--fiqs8s'
+'xn--fiqs8s', null

data/test/unit/domain_test.rb

@@ -1,4 +1,4 @@
-require 'test_helper'
+require "test_helper"
 
 class PublicSuffix::DomainTest < Minitest::Unit::TestCase
 
@@ -7,25 +7,17 @@ class PublicSuffix::DomainTest < Minitest::Unit::TestCase
   end
 
   # Tokenizes given input into labels.
-  def test_self_domain_to_labels
-    assert_equal  %w( com live spaces someone ),
-                  PublicSuffix::Domain.domain_to_labels("someone.spaces.live.com")
-    assert_equal  %w( com zoho wiki leontina23samiko ),
-                  PublicSuffix::Domain.domain_to_labels("leontina23samiko.wiki.zoho.com")
+  def test_self_name_to_labels
+    assert_equal  %w( someone spaces live com ),
+                  PublicSuffix::Domain.name_to_labels("someone.spaces.live.com")
+    assert_equal  %w( leontina23samiko wiki zoho com ),
+                  PublicSuffix::Domain.name_to_labels("leontina23samiko.wiki.zoho.com")
   end
 
   # Converts input into String.
-  def test_self_domain_to_labels_converts_input_to_string
-    assert_equal  %w( com live spaces someone ),
-                  PublicSuffix::Domain.domain_to_labels(:"someone.spaces.live.com")
-  end
-
-  # Ignores trailing .
-  def test_self_domain_to_labels_ignores_trailing_dot
-    assert_equal  %w( com live spaces someone ),
-                  PublicSuffix::Domain.domain_to_labels("someone.spaces.live.com")
-    assert_equal  %w( com live spaces someone ),
-                  PublicSuffix::Domain.domain_to_labels(:"someone.spaces.live.com")
+  def test_self_name_to_labels_converts_input_to_string
+    assert_equal  %w( someone spaces live com ),
+                  PublicSuffix::Domain.name_to_labels(:"someone.spaces.live.com")
   end
 
 
@@ -85,86 +77,28 @@ class PublicSuffix::DomainTest < Minitest::Unit::TestCase
 
   def test_domain
     assert_equal nil, @klass.new("com").domain
-    assert_equal nil, @klass.new("qqq").domain
+    assert_equal nil, @klass.new("tldnotlisted").domain
     assert_equal "google.com", @klass.new("com", "google").domain
-    assert_equal "google.qqq", @klass.new("qqq", "google").domain
+    assert_equal "google.tldnotlisted", @klass.new("tldnotlisted", "google").domain
     assert_equal "google.com", @klass.new("com", "google", "www").domain
-    assert_equal "google.qqq", @klass.new("qqq", "google", "www").domain
+    assert_equal "google.tldnotlisted", @klass.new("tldnotlisted", "google", "www").domain
   end
 
   def test_subdomain
     assert_equal nil, @klass.new("com").subdomain
-    assert_equal nil, @klass.new("qqq").subdomain
+    assert_equal nil, @klass.new("tldnotlisted").subdomain
     assert_equal nil, @klass.new("com", "google").subdomain
-    assert_equal nil, @klass.new("qqq", "google").subdomain
+    assert_equal nil, @klass.new("tldnotlisted", "google").subdomain
     assert_equal "www.google.com", @klass.new("com", "google", "www").subdomain
-    assert_equal "www.google.qqq", @klass.new("qqq", "google", "www").subdomain
-  end
-
-  def test_rule
-    assert_equal nil, @klass.new("qqq").rule
-    assert_equal PublicSuffix::Rule.factory("com"), @klass.new("com").rule
-    assert_equal PublicSuffix::Rule.factory("com"), @klass.new("com", "google").rule
-    assert_equal PublicSuffix::Rule.factory("com"), @klass.new("com", "google", "www").rule
+    assert_equal "www.google.tldnotlisted", @klass.new("tldnotlisted", "google", "www").subdomain
   end
 
 
   def test_domain_question
-    assert  @klass.new("com", "google").domain?
-    assert  @klass.new("qqq", "google").domain?
-    assert  @klass.new("com", "google", "www").domain?
     assert !@klass.new("com").domain?
-  end
-
-  def test_subdomain_question
-    assert  @klass.new("com", "google", "www").subdomain?
-    assert  @klass.new("qqq", "google", "www").subdomain?
-    assert !@klass.new("com").subdomain?
-    assert !@klass.new("com", "google").subdomain?
-  end
-
-  def test_is_a_domain_question
-    assert  @klass.new("com", "google").is_a_domain?
-    assert  @klass.new("qqq", "google").is_a_domain?
-    assert !@klass.new("com", "google", "www").is_a_domain?
-    assert !@klass.new("com").is_a_domain?
-  end
-
-  def test_is_a_subdomain_question
-    assert  @klass.new("com", "google", "www").is_a_subdomain?
-    assert  @klass.new("qqq", "google", "www").is_a_subdomain?
-    assert !@klass.new("com").is_a_subdomain?
-    assert !@klass.new("com", "google").is_a_subdomain?
-  end
-
-  def test_valid_question
-    assert !@klass.new("com").valid?
-    assert  @klass.new("com", "example").valid?
-    assert  @klass.new("com", "example", "www").valid?
-
-    # not-assigned
-    assert !@klass.new("qqq").valid?
-    assert !@klass.new("qqq", "example").valid?
-    assert !@klass.new("qqq", "example", "www").valid?
-
-    # not-allowed
-    assert !@klass.new("ke").valid?
-    assert !@klass.new("ke", "example").valid?
-    assert  @klass.new("ke", "example", "www").valid?
-  end
-
-  def test_valid_domain_question
-    assert  @klass.new("com", "google").valid_domain?
-    assert !@klass.new("qqq", "google").valid_domain?
-    assert  @klass.new("com", "google", "www").valid_domain?
-    assert !@klass.new("com").valid_domain?
-  end
-
-  def test_valid_subdomain_question
-    assert  @klass.new("com", "google", "www").valid_subdomain?
-    assert !@klass.new("qqq", "google", "www").valid_subdomain?
-    assert !@klass.new("com").valid_subdomain?
-    assert !@klass.new("com", "google").valid_subdomain?
+    assert  @klass.new("com", "example").domain?
+    assert  @klass.new("com", "example", "www").domain?
+    assert  @klass.new("tldnotlisted", "example").domain?
   end
 
 end