domain_prefix 0.1.0

data/domain_prefix.gemspec

@@ -0,0 +1,47 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = "domain_prefix"
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["tadman"]
+  s.date = "2012-02-03"
+  s.description = "A library to extract information about top-level domain and registered name from generic and international domain names"
+  s.email = "github@tadman.ca"
+  s.extra_rdoc_files = [
+    "LICENSE",
+    "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+    "LICENSE",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "data/effective_tld_names.dat",
+    "domain_prefix.gemspec",
+    "lib/domain_prefix.rb",
+    "test/helper.rb",
+    "test/sample/README",
+    "test/sample/test.txt",
+    "test/test_tldifier.rb"
+  ]
+  s.homepage = "http://github.com/twg/domain_prefix"
+  s.require_paths = ["lib"]
+  s.rubygems_version = "1.8.11"
+  s.summary = "Domain Prefix Extraction Library"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end
+

data/lib/domain_prefix.rb

@@ -0,0 +1,112 @@
+# encoding: UTF-8
+
+module DomainPrefix
+  class TreeHash < Hash
+    def initialize
+      super do |h, k|
+        h[k] = TreeHash.new
+      end
+    end
+    
+    def find_domain(domain)
+      domain.split('.').inject(self) do |h, component|
+        h and h.key?(component) ? h[component] : nil
+      end
+    end
+  end
+  
+  TLDIFIER_SOURCE_FILE = File.expand_path(File.join('..', 'data', 'effective_tld_names.dat'), File.dirname(__FILE__))
+
+  TLD_SET = File.read(TLDIFIER_SOURCE_FILE).split(/\n/).collect do |line|
+    line.sub(%r[//.*], '').sub(/\s+$/, '')
+  end.reject(&:empty?).freeze
+  
+  TLD_NAMES = TLD_SET.sort_by do |d|
+    [ -d.length, d ]
+  end.freeze
+  
+  TLD_TREE = TLD_NAMES.inject(TreeHash.new) do |h, name|
+    name.split('.').reverse.inject(h) do |_h, component|
+      case (component)
+      when '*'
+        _h
+      when /!(.*)/
+        _h[$1]
+      else
+        _h[component]
+      end
+    end
+
+    h
+  end.freeze
+
+  PREFIX_SPEC = Regexp.new(
+    '^(' + TLD_NAMES.collect do |d|
+      Regexp.escape(d).sub(/^\\\*\\\./, '')
+    end.join('|') + ')$'
+  ).freeze
+  
+  ALLOWED_DOMAIN_PREFIXES = Hash[
+    TLD_NAMES.select do |d|
+      d.match(/^\!/)
+    end.collect do |d|
+      [ d.sub(/^\!/, ''), true ]
+    end
+  ].freeze
+
+  DOMAIN_PREFIX_SPEC = Regexp.new(
+    '^(?:[^\.]+\.)*?(([^\.]+)\.(' + TLD_NAMES.collect do |d|
+      Regexp.escape(d).sub(/^\\\*\\\./, '[^\.]+\.')
+    end.join('|') + '))$'
+  ).freeze
+  
+  NONPUBLIC_TLD = {
+    'local' => true
+  }.freeze
+  
+  def rfc3492_canonical_domain(domain)
+    # FIX: Full implementation of http://www.ietf.org/rfc/rfc3492.txt required
+    domain and domain.downcase
+  end
+
+  def registered_domain(domain)
+    m = DOMAIN_PREFIX_SPEC.match(rfc3492_canonical_domain(domain))
+    
+    return unless (m)
+
+    domain = m[1]
+    suffix = m[3]
+    
+    return if (NONPUBLIC_TLD[suffix])
+    return if (PREFIX_SPEC.match(domain) and !ALLOWED_DOMAIN_PREFIXES[domain])
+    
+    domain
+  end
+
+  def public_suffix(domain)
+    m = DOMAIN_PREFIX_SPEC.match(rfc3492_canonical_domain(domain))
+    
+    return unless (m)
+    
+    domain = m[1]
+    suffix = m[3]
+    
+    return if (PREFIX_SPEC.match(domain) and !ALLOWED_DOMAIN_PREFIXES[domain])
+    
+    suffix
+  end
+
+  def tld(domain)
+    suffix = public_suffix(rfc3492_canonical_domain(domain))
+    
+    suffix and suffix.split(/\./).last
+  end
+  
+  def name(domain)
+    m = DOMAIN_PREFIX_SPEC.match(rfc3492_canonical_domain(domain))
+    
+    m and m[2]
+  end
+  
+  extend self
+end

data/test/helper.rb

@@ -0,0 +1,35 @@
+require 'rubygems'
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'domain_prefix'
+
+class Test::Unit::TestCase
+  def assert_mapping(map)
+    result_map = map.inject({ }) do |h, (k,v)|
+      h[k] = yield(k)
+      h
+    end
+    
+    differences = result_map.inject([ ]) do |a, (k,v)|
+      if (v != map[k])
+        a << k
+      end
+
+      a
+    end
+    
+    assert_equal(map, result_map, differences.collect { |s| "Input: #{s.inspect}\n  Expected: #{map[s].inspect}\n  Result:   #{result_map[s].inspect}\n" }.join(''))
+  end
+  
+  def sample_data(file)
+    File.read(
+      File.expand_path(
+        File.join('sample', file),
+        File.dirname(__FILE__)
+      )
+    )
+  end
+end

data/test/sample/README

@@ -0,0 +1 @@
+Sample data pulled from http://publicsuffix.org/list/test.txt

data/test/sample/test.txt

@@ -0,0 +1,76 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+# NULL input.
+checkPublicSuffix(NULL, NULL);
+# Mixed case.
+checkPublicSuffix('COM', NULL);
+checkPublicSuffix('example.COM', 'example.com');
+checkPublicSuffix('WwW.example.COM', 'example.com');
+# Leading dot.
+checkPublicSuffix('.com', NULL);
+checkPublicSuffix('.example', NULL);
+checkPublicSuffix('.example.com', NULL);
+checkPublicSuffix('.example.example', NULL);
+# Unlisted TLD.
+checkPublicSuffix('example', NULL);
+checkPublicSuffix('example.example', NULL);
+checkPublicSuffix('b.example.example', NULL);
+checkPublicSuffix('a.b.example.example', NULL);
+# Listed, but non-Internet, TLD.
+checkPublicSuffix('local', NULL);
+checkPublicSuffix('example.local', NULL);
+checkPublicSuffix('b.example.local', NULL);
+checkPublicSuffix('a.b.example.local', NULL);
+# TLD with only 1 rule.
+checkPublicSuffix('biz', NULL);
+checkPublicSuffix('domain.biz', 'domain.biz');
+checkPublicSuffix('b.domain.biz', 'domain.biz');
+checkPublicSuffix('a.b.domain.biz', 'domain.biz');
+# TLD with some 2-level rules.
+checkPublicSuffix('com', NULL);
+checkPublicSuffix('example.com', 'example.com');
+checkPublicSuffix('b.example.com', 'example.com');
+checkPublicSuffix('a.b.example.com', 'example.com');
+checkPublicSuffix('uk.com', NULL);
+checkPublicSuffix('example.uk.com', 'example.uk.com');
+checkPublicSuffix('b.example.uk.com', 'example.uk.com');
+checkPublicSuffix('a.b.example.uk.com', 'example.uk.com');
+checkPublicSuffix('test.ac', 'test.ac');
+# TLD with only 1 (wildcard) rule.
+checkPublicSuffix('cy', NULL);
+checkPublicSuffix('c.cy', NULL);
+checkPublicSuffix('b.c.cy', 'b.c.cy');
+checkPublicSuffix('a.b.c.cy', 'b.c.cy');
+# More complex TLD.
+checkPublicSuffix('jp', NULL);
+checkPublicSuffix('test.jp', 'test.jp');
+checkPublicSuffix('www.test.jp', 'test.jp');
+checkPublicSuffix('ac.jp', NULL);
+checkPublicSuffix('test.ac.jp', 'test.ac.jp');
+checkPublicSuffix('www.test.ac.jp', 'test.ac.jp');
+checkPublicSuffix('kyoto.jp', NULL);
+checkPublicSuffix('c.kyoto.jp', NULL);
+checkPublicSuffix('b.c.kyoto.jp', 'b.c.kyoto.jp');
+checkPublicSuffix('a.b.c.kyoto.jp', 'b.c.kyoto.jp');
+checkPublicSuffix('pref.kyoto.jp', 'pref.kyoto.jp');	# Exception rule.
+checkPublicSuffix('www.pref.kyoto.jp', 'pref.kyoto.jp');	# Exception rule.
+checkPublicSuffix('city.kyoto.jp', 'city.kyoto.jp');	# Exception rule.
+checkPublicSuffix('www.city.kyoto.jp', 'city.kyoto.jp');	# Exception rule.
+# TLD with a wildcard rule and exceptions.
+checkPublicSuffix('om', NULL);
+checkPublicSuffix('test.om', NULL);
+checkPublicSuffix('b.test.om', 'b.test.om');
+checkPublicSuffix('a.b.test.om', 'b.test.om');
+checkPublicSuffix('songfest.om', 'songfest.om');
+checkPublicSuffix('www.songfest.om', 'songfest.om');
+# US K12.
+checkPublicSuffix('us', NULL);
+checkPublicSuffix('test.us', 'test.us');
+checkPublicSuffix('www.test.us', 'test.us');
+checkPublicSuffix('ak.us', NULL);
+checkPublicSuffix('test.ak.us', 'test.ak.us');
+checkPublicSuffix('www.test.ak.us', 'test.ak.us');
+checkPublicSuffix('k12.ak.us', NULL);
+checkPublicSuffix('test.k12.ak.us', 'test.k12.ak.us');
+checkPublicSuffix('www.test.k12.ak.us', 'test.k12.ak.us');

data/test/test_tldifier.rb

@@ -0,0 +1,49 @@
+require 'helper'
+
+class TestDomainPrefix < Test::Unit::TestCase
+  def test_initialization
+  end
+  
+  def test_examples
+    assert_mapping(
+      'com' => [ nil, nil ],
+      'example.com' => %w[ example.com com ],
+      'uk.com' => [ nil, nil ],
+      'example.uk.com' => %w[ example.uk.com uk.com ],
+      'example.ca' =>  %w[ example.ca ca ],
+      'example.on.ca' =>  %w[ example.on.ca on.ca ],
+      'example.gc.ca' =>  %w[ example.gc.ca gc.ca ],
+      'example.co.uk' =>  %w[ example.co.uk co.uk ],
+      'example.au' => [ nil, nil ],
+      'example.com.au' => %w[ example.com.au com.au ],
+      'example.bar.jp' => %w[ bar.jp jp ],
+      'example.bar.hokkaido.jp' =>%w[ example.bar.hokkaido.jp bar.hokkaido.jp ],
+      'example.metro.tokyo.jp' => %w[ example.metro.tokyo.jp metro.tokyo.jp ]
+    ) do |domain|
+      [
+        DomainPrefix.registered_domain(domain),
+        DomainPrefix.public_suffix(domain)
+      ]
+    end
+  end
+  
+  def test_public_suffix_samples
+    sample_data('test.txt').split(/\n/).collect do |line|
+      case (line)
+      when /checkPublicSuffix\((\S+),\s*(\S+)\)/
+        [ $1, $2 ].collect do |part|
+          case (part)
+          when 'NULL'
+            nil
+          else
+            part.gsub(/'/, '')
+          end
+        end
+      else
+        nil
+      end
+    end.each do |domain, expected|
+      assert_equal expected, DomainPrefix.registered_domain(domain), "#{domain.inspect} -> #{expected.inspect}"
+    end
+  end
+end

metadata

@@ -0,0 +1,59 @@
+--- !ruby/object:Gem::Specification
+name: domain_prefix
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- tadman
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-02-03 00:00:00.000000000 Z
+dependencies: []
+description: A library to extract information about top-level domain and registered
+  name from generic and international domain names
+email: github@tadman.ca
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE
+- README.rdoc
+files:
+- .document
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- data/effective_tld_names.dat
+- domain_prefix.gemspec
+- lib/domain_prefix.rb
+- test/helper.rb
+- test/sample/README
+- test/sample/test.txt
+- test/test_tldifier.rb
+homepage: http://github.com/twg/domain_prefix
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.11
+signing_key: 
+specification_version: 3
+summary: Domain Prefix Extraction Library
+test_files: []