domain_name 0.0.0

data/lib/domain_name/etld_data.rb.erb

@@ -0,0 +1,7 @@
+class DomainName
+  def self.etld_data
+    @@etld_data ||= {
+<% etld_data.each_pair { |key, value| %>      <%= key.inspect %> => <%= value.inspect %>,
+<% } %>    }
+  end
+end

data/lib/domain_name/punycode.rb

@@ -0,0 +1,182 @@
+# -*- coding: utf-8 -*-
+#
+# punycode.rb - PunyCode encoder for the Domain Name library
+#
+# Copyright (C) 2011 Akinori MUSHA, All rights reserved.
+#
+# Ported from puny.c, a part of VeriSign XCode (encode/decode) IDN
+# Library.
+#
+# Copyright (C) 2000-2002 Verisign Inc., All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or
+# without modification, are permitted provided that the following
+# conditions are met:
+#
+#  1) Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#
+#  2) Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in
+#     the documentation and/or other materials provided with the
+#     distribution.
+#
+#  3) Neither the name of the VeriSign Inc. nor the names of its
+#     contributors may be used to endorse or promote products derived
+#     from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# This software is licensed under the BSD open source license. For more
+# information visit www.opensource.org.
+#
+# Authors:
+#  John Colosi (VeriSign)
+#  Srikanth Veeramachaneni (VeriSign)
+#  Nagesh Chigurupati (Verisign)
+#  Praveen Srinivasan(Verisign)
+
+class DomainName
+  module Punycode
+    BASE = 36
+    TMIN = 1
+    TMAX = 26
+    SKEW = 38
+    DAMP = 700
+    INITIAL_BIAS = 72
+    INITIAL_N = 0x80
+    DELIMITER = '-'
+
+    # The maximum value of an DWORD variable
+    MAXINT = (1 << 64) - 1
+
+    # Used in the calculation of bias:
+    LOBASE = BASE - TMIN
+
+    # Used in the calculation of bias:
+    CUTOFF = LOBASE * TMAX / 2
+
+    class Error < StandardError; end
+    class BufferOverflowError < Error; end
+
+    # Returns the basic code point whose value (when used for
+    # representing integers) is d, which must be in the range 0 to
+    # BASE-1.  The lowercase form is used unless flag is true, in
+    # which case the uppercase form is used.  The behavior is
+    # undefined if flag is nonzero and digit d has no uppercase form.
+    def encode_digit(d, flag)
+      (d + 22 + (d < 26 ? 75 : 0) - (flag ? (1 << 5) : 0)).chr
+      #  0..25 map to ASCII a..z or A..Z
+      # 26..35 map to ASCII 0..9
+    end
+    module_function :encode_digit
+
+    # Main encode function
+    def encode(string)
+      input = string.unpack('U*')
+      output = ''
+
+      # Initialize the state
+      n = INITIAL_N
+      delta = 0
+      bias = INITIAL_BIAS;
+
+      # Handle the basic code points
+      input.each { |cp| output << cp.chr if cp < 0x80 }
+
+      h = b = output.length
+
+      # h is the number of code points that have been handled, b is the
+      # number of basic code points, and out is the number of characters
+      # that have been output.
+
+      output << DELIMITER if b > 0
+
+      # Main encoding loop
+
+      while h < input.length
+        # All non-basic code points < n have been handled already.  Find
+        # the next larger one
+
+        m = MAXINT
+        input.each { |cp|
+          m = cp if (n...m) === cp
+        }
+
+        # Increase delta enough to advance the decoder's <n,i> state to
+        # <m,0>, but guard against overflow
+
+        if m - n > (MAXINT - delta) / (h + 1)
+          raise BufferOverflowError
+        end
+        delta += (m - n) * (h + 1)
+        n = m
+
+        input.each { |cp|
+          # AMC-ACE-Z can use this simplified version instead
+          if cp < n && (delta += 1) == 0
+            raise BufferOverflowError
+          end
+
+          if cp == n
+            # Represent delta as a generalized variable-length integer
+            q = delta
+            k = BASE
+            loop {
+              t = k <= bias ? TMIN : k - bias >= TMAX ? TMAX : k - bias;
+              break if q < t
+              output << encode_digit(t + (q - t) % (BASE - t), false)
+              q = (q - t) / (BASE - t)
+              k += BASE
+            }
+
+            output << encode_digit(q, false)
+
+            # Adapt the bias
+            delta = h == b ? delta / DAMP : delta >> 1
+            delta += delta / (h + 1)
+            bias = 0
+            while delta > CUTOFF
+              delta /= LOBASE
+              bias += BASE
+            end
+            bias += (LOBASE + 1) * delta / (delta + SKEW)
+
+            delta = 0
+            h += 1
+          end
+        }
+
+        delta += 1
+        n += 1
+      end
+
+      output
+    end
+    module_function :encode
+
+    def encode_hostname(hostname)
+      hostname.match(/[^\x00-\x7f]/) or return hostname
+
+      hostname.split('.').map { |name|
+        if name.match(/[^\x00-\x7f]/)
+          'xn--' << encode(name)
+        else
+          name
+        end
+      }.join('.')
+    end
+    module_function :encode_hostname
+  end
+end

data/test/helper.rb

@@ -0,0 +1,18 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'domain_name'
+
+class Test::Unit::TestCase
+end

data/test/test_domain_name-punycode.rb

@@ -0,0 +1,96 @@
+require 'helper'
+
+class TestDomainName < Test::Unit::TestCase
+  should "encode labels just as listed in RFC 3492 #7.1 (slightly modified)" do
+    [
+      ['(A) Arabic (Egyptian)',
+        [0x0644, 0x064A, 0x0647, 0x0645, 0x0627, 0x0628, 0x062A, 0x0643, 0x0644,
+          0x0645, 0x0648, 0x0634, 0x0639, 0x0631, 0x0628, 0x064A, 0x061F],
+        'egbpdaj6bu4bxfgehfvwxn'],
+      ['(B) Chinese (simplified)',
+        [0x4ED6, 0x4EEC, 0x4E3A, 0x4EC0, 0x4E48, 0x4E0D, 0x8BF4, 0x4E2D, 0x6587],
+        'ihqwcrb4cv8a8dqg056pqjye'],
+      ['(C) Chinese (traditional)',
+        [0x4ED6, 0x5011, 0x7232, 0x4EC0, 0x9EBD, 0x4E0D, 0x8AAA, 0x4E2D, 0x6587],
+        'ihqwctvzc91f659drss3x8bo0yb'],
+      ['(D) Czech: Pro<ccaron>prost<ecaron>nemluv<iacute><ccaron>esky',
+        [0x0050, 0x0072, 0x006F, 0x010D, 0x0070, 0x0072, 0x006F, 0x0073, 0x0074,
+          0x011B, 0x006E, 0x0065, 0x006D, 0x006C, 0x0075, 0x0076, 0x00ED, 0x010D,
+          0x0065, 0x0073, 0x006B, 0x0079],
+        'Proprostnemluvesky-uyb24dma41a'],
+      ['(E) Hebrew',
+        [0x05DC, 0x05DE, 0x05D4, 0x05D4, 0x05DD, 0x05E4, 0x05E9, 0x05D5, 0x05D8,
+          0x05DC, 0x05D0, 0x05DE, 0x05D3, 0x05D1, 0x05E8, 0x05D9, 0x05DD, 0x05E2,
+          0x05D1, 0x05E8, 0x05D9, 0x05EA],
+        '4dbcagdahymbxekheh6e0a7fei0b'],
+      ['(F) Hindi (Devanagari)',
+      [0x092F, 0x0939, 0x0932, 0x094B, 0x0917, 0x0939, 0x093F, 0x0928, 0x094D,
+          0x0926, 0x0940, 0x0915, 0x094D, 0x092F, 0x094B, 0x0902, 0x0928, 0x0939,
+          0x0940, 0x0902, 0x092C, 0x094B, 0x0932, 0x0938, 0x0915, 0x0924, 0x0947,
+          0x0939, 0x0948, 0x0902],
+        'i1baa7eci9glrd9b2ae1bj0hfcgg6iyaf8o0a1dig0cd'],
+      ['(G) Japanese (kanji and hiragana)',
+        [0x306A, 0x305C, 0x307F, 0x3093, 0x306A, 0x65E5, 0x672C, 0x8A9E, 0x3092,
+          0x8A71, 0x3057, 0x3066, 0x304F, 0x308C, 0x306A, 0x3044, 0x306E, 0x304B],
+        'n8jok5ay5dzabd5bym9f0cm5685rrjetr6pdxa'],
+      ['(H) Korean (Hangul syllables)',
+        [0xC138, 0xACC4, 0xC758, 0xBAA8, 0xB4E0, 0xC0AC, 0xB78C, 0xB4E4, 0xC774,
+          0xD55C, 0xAD6D, 0xC5B4, 0xB97C, 0xC774, 0xD574, 0xD55C, 0xB2E4, 0xBA74,
+          0xC5BC, 0xB9C8, 0xB098, 0xC88B, 0xC744, 0xAE4C],
+        '989aomsvi5e83db1d2a355cv1e0vak1dwrv93d5xbh15a0dt30a5j' <<
+        'psd879ccm6fea98c'],
+      ['(I) Russian (Cyrillic)',
+        [0x043F, 0x043E, 0x0447, 0x0435, 0x043C, 0x0443, 0x0436, 0x0435, 0x043E,
+          0x043D, 0x0438, 0x043D, 0x0435, 0x0433, 0x043E, 0x0432, 0x043E, 0x0440,
+          0x044F, 0x0442, 0x043F, 0x043E, 0x0440, 0x0443, 0x0441, 0x0441, 0x043A,
+          0x0438],
+        'b1abfaaepdrnnbgefbadotcwatmq2g4l'],
+      ['(J) Spanish: Porqu<eacute>nopuedensimplementehablarenEspa<ntilde>ol',
+        [0x0050, 0x006F, 0x0072, 0x0071, 0x0075, 0x00E9, 0x006E, 0x006F, 0x0070,
+          0x0075, 0x0065, 0x0064, 0x0065, 0x006E, 0x0073, 0x0069, 0x006D, 0x0070,
+          0x006C, 0x0065, 0x006D, 0x0065, 0x006E, 0x0074, 0x0065, 0x0068, 0x0061,
+          0x0062, 0x006C, 0x0061, 0x0072, 0x0065, 0x006E, 0x0045, 0x0073, 0x0070,
+          0x0061, 0x00F1, 0x006F, 0x006C],
+        'PorqunopuedensimplementehablarenEspaol-fmd56a'],
+      ['(K) Vietnamese: T<adotbelow>isaoh<odotbelow>kh<ocirc>ngth<ecirchookabove>ch' <<
+        '<ihookabove>n<oacute>iti<ecircacute>ngVi<ecircdotbelow>t',
+        [0x0054, 0x1EA1, 0x0069, 0x0073, 0x0061, 0x006F, 0x0068, 0x1ECD, 0x006B,
+          0x0068, 0x00F4, 0x006E, 0x0067, 0x0074, 0x0068, 0x1EC3, 0x0063, 0x0068,
+          0x1EC9, 0x006E, 0x00F3, 0x0069, 0x0074, 0x0069, 0x1EBF, 0x006E, 0x0067,
+          0x0056, 0x0069, 0x1EC7, 0x0074],
+        'TisaohkhngthchnitingVit-kjcr8268qyxafd2f1b9g'],
+      ['(L) 3<nen>B<gumi><kinpachi><sensei>',
+        [0x0033, 0x5E74, 0x0042, 0x7D44, 0x91D1, 0x516B, 0x5148, 0x751F],
+        '3B-ww4c5e180e575a65lsy2b'],
+      ['(M) <amuro><namie>-with-SUPER-MONKEYS',
+        [0x5B89, 0x5BA4, 0x5948, 0x7F8E, 0x6075, 0x002D, 0x0077, 0x0069, 0x0074,
+          0x0068, 0x002D, 0x0053, 0x0055, 0x0050, 0x0045, 0x0052, 0x002D, 0x004D,
+          0x004F, 0x004E, 0x004B, 0x0045, 0x0059, 0x0053],
+        '-with-SUPER-MONKEYS-pc58ag80a8qai00g7n9n'],
+      ['(N) Hello-Another-Way-<sorezore><no><basho>',
+       [0x0048, 0x0065, 0x006C, 0x006C, 0x006F, 0x002D, 0x0041, 0x006E, 0x006F,
+          0x0074, 0x0068, 0x0065, 0x0072, 0x002D, 0x0057, 0x0061, 0x0079, 0x002D,
+          0x305D, 0x308C, 0x305E, 0x308C, 0x306E, 0x5834, 0x6240],
+        'Hello-Another-Way--fc4qua05auwb3674vfr0b'],
+      ['(O) <hitotsu><yane><no><shita>2',
+        [0x3072, 0x3068, 0x3064, 0x5C4B, 0x6839, 0x306E, 0x4E0B, 0x0032],
+        '2-u9tlzr9756bt3uc0v'],
+      ['(P) Maji<de>Koi<suru>5<byou><mae>',
+        [0x004D, 0x0061, 0x006A, 0x0069, 0x3067, 0x004B, 0x006F, 0x0069, 0x3059,
+          0x308B, 0x0035, 0x79D2, 0x524D],
+        'MajiKoi5-783gue6qz075azm5e'],
+      ['(Q) <pafii>de<runba>',
+        [0x30D1, 0x30D5, 0x30A3, 0x30FC, 0x0064, 0x0065, 0x30EB, 0x30F3, 0x30D0],
+        'de-jg4avhby1noc0d'],
+      ['(R) <sono><supiido><de>',
+        [0x305D, 0x306E, 0x30B9, 0x30D4, 0x30FC, 0x30C9, 0x3067],
+        'd9juau41awczczp'],
+      ['(S) -> $1.00 <-',
+         [0x002D, 0x003E, 0x0020, 0x0024, 0x0031, 0x002E, 0x0030, 0x0030, 0x0020,
+          0x003C, 0x002D],
+        '-> $1.00 <--']
+    ].each { |title, cps, punycode|
+      assert_equal punycode, DomainName::Punycode.encode(cps.pack('U*')), title
+    }
+  end
+end

data/test/test_domain_name.rb

@@ -0,0 +1,89 @@
+require 'helper'
+
+class TestDomainName < Test::Unit::TestCase
+  should "raise ArgumentError if hostname starts with a dot" do
+    [
+      # Leading dot.
+      '.com',
+      '.example',
+      '.example.com',
+      '.example.example',
+    ].each { |hostname|
+      assert_raises(ArgumentError) { DomainName.new(hostname) }
+    }
+  end
+
+  should "parse canonical domain names correctly" do
+    [
+      # Mixed case.
+      ['COM', nil],
+      ['example.COM', 'example.com'],
+      ['WwW.example.COM', 'example.com'],
+      # Unlisted TLD.
+      ['example', 'example'],
+      ['example.example', 'example.example'],
+      ['b.example.example', 'example.example'],
+      ['a.b.example.example', 'example.example'],
+      # Listed, but non-Internet, TLD.
+      ['local', 'local'],
+      ['example.local', 'example.local'],
+      ['b.example.local', 'example.local'],
+      ['a.b.example.local', 'example.local'],
+      # TLD with only 1 rule.
+      ['biz', nil],
+      ['domain.biz', 'domain.biz'],
+      ['b.domain.biz', 'domain.biz'],
+      ['a.b.domain.biz', 'domain.biz'],
+      # TLD with some 2-level rules.
+      ['com', nil],
+      ['example.com', 'example.com'],
+      ['b.example.com', 'example.com'],
+      ['a.b.example.com', 'example.com'],
+      ['uk.com', nil],
+      ['example.uk.com', 'example.uk.com'],
+      ['b.example.uk.com', 'example.uk.com'],
+      ['a.b.example.uk.com', 'example.uk.com'],
+      ['test.ac', 'test.ac'],
+      # TLD with only 1 (wildcard) rule.
+      ['cy', nil],
+      ['c.cy', nil],
+      ['b.c.cy', 'b.c.cy'],
+      ['a.b.c.cy', 'b.c.cy'],
+      # More complex TLD.
+      ['jp', nil],
+      ['test.jp', 'test.jp'],
+      ['www.test.jp', 'test.jp'],
+      ['ac.jp', nil],
+      ['test.ac.jp', 'test.ac.jp'],
+      ['www.test.ac.jp', 'test.ac.jp'],
+      ['kyoto.jp', nil],
+      ['c.kyoto.jp', nil],
+      ['b.c.kyoto.jp', 'b.c.kyoto.jp'],
+      ['a.b.c.kyoto.jp', 'b.c.kyoto.jp'],
+      ['pref.kyoto.jp', 'pref.kyoto.jp'],	# Exception rule
+      ['www.pref.kyoto.jp', 'pref.kyoto.jp'],	# Exception rule.
+      ['city.kyoto.jp', 'city.kyoto.jp'],	# Exception rule.
+      ['www.city.kyoto.jp', 'city.kyoto.jp'],	# Exception rule.
+      # TLD with a wildcard rule and exceptions.
+      ['om', nil],
+      ['test.om', nil],
+      ['b.test.om', 'b.test.om'],
+      ['a.b.test.om', 'b.test.om'],
+      ['songfest.om', 'songfest.om'],
+      ['www.songfest.om', 'songfest.om'],
+      # US K12.
+      ['us', nil],
+      ['test.us', 'test.us'],
+      ['www.test.us', 'test.us'],
+      ['ak.us', nil],
+      ['test.ak.us', 'test.ak.us'],
+      ['www.test.ak.us', 'test.ak.us'],
+      ['k12.ak.us', nil],
+      ['test.k12.ak.us', 'test.k12.ak.us'],
+      ['www.test.k12.ak.us', 'test.k12.ak.us'],
+    ].each { |hostname, domain|
+      dn = DomainName.new(hostname)
+      assert_equal(domain, dn.domain)
+    }
+  end
+end

data/tool/gen_etld_data.rb

@@ -0,0 +1,61 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'pathname'
+$basedir = Pathname.new(__FILE__).dirname.parent
+$LOAD_PATH.unshift $basedir + 'lib'
+require 'domain_name'
+require 'set'
+require 'erb'
+
+def main
+  dat_file = $basedir + 'data' + 'effective_tld_names.dat'
+  dir      = $basedir + 'lib' + 'domain_name'
+  erb_file = dir + 'etld_data.rb.erb'
+  rb_file  = dir + 'etld_data.rb'
+
+  File.open(dat_file, 'r:utf-8') { |dat|
+    etld_data = parse(dat)
+    File.open(rb_file, 'w:utf-8') { |rb|
+      File.open(erb_file, 'r:utf-8') { |erb|
+        rb.print ERB.new(erb.read).result(binding)
+      }
+    }
+  }
+end
+
+def normalize_hostname(domain)
+  DomainName.normalize(domain)
+end
+
+def parse(f)
+  {}.tap { |table|
+    tlds = Set[]
+    f.each_line { |line|
+      line.sub!(%r{//.*}, '')
+      line.strip!
+      next if line.empty?
+      case line
+      when /^local$/
+        # ignore .local
+        next
+      when /^([^!*]+)$/
+        domain = normalize_hostname($1)
+        value = 0
+      when /^\*\.([^!*]+)$/
+        domain = normalize_hostname($1)
+        value = -1
+      when /^\!([^!*]+)$/
+        domain = normalize_hostname($1)
+        value = 1
+      else
+        raise "syntax error: #{line}"
+      end
+      tld = domain.match(/(?:^|\.)([^.]+)$/)[1]
+      table[tld] ||= 1
+      table[domain] = value
+    }
+  }
+end
+
+main()