uri-idna 0.1.0 → 0.2.0

data/lib/uri/idna/idna2008/processing.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+require_relative "options"
+
+module URI
+  module IDNA
+    module IDNA2008
+      class Processing < BaseProcessing
+        private
+
+        def options_class
+          Options
+        end
+
+        def validate(label)
+          return if label.empty?
+
+          Validation::Label.check_nfc(label)
+          if options.check_hyphens?
+            Validation::Label.check_hyphen34(label)
+          else
+            Validation::Label.check_ace_prefix(label)
+          end
+          Validation::Label.check_leading_combining(label) if options.leading_combining?
+
+          label.each_codepoint.with_index do |cp, pos|
+            begin
+              next if Validation::Codepoint.check_contextj(label, cp, pos)
+            rescue InvalidCodepointContextError => e
+              next unless options.check_joiners?
+
+              raise e
+            end
+
+            begin
+              next if Validation::Codepoint.check_contexto(label, cp, pos)
+            rescue InvalidCodepointContextError => e
+              next unless options.check_others?
+
+              raise e
+            end
+
+            Validation::Codepoint.check_idna_validity(label, cp, pos)
+          end
+          Validation::Bidi.call(label) if check_bidi?
+        end
+
+        def check_bidi?
+          return @check_bidi if instance_variable_defined?(:@check_bidi)
+
+          @check_bidi = options.check_bidi? && Validation::Bidi.check?(domain_name)
+        end
+
+        def punycode_decode(label)
+          return label unless label.start_with?(ACE_PREFIX)
+
+          raise Error, "A-label must not end with a hyphen" if label[-1] == "-"
+
+          super
+        end
+      end
+
+      # https://datatracker.ietf.org/doc/html/rfc5891#section-4
+      class Registration < Processing
+        def initialize(alabel: nil, ulabel: nil, **options)
+          raise ArgumentError, "Provide alabel or ulabel" if alabel.nil? && ulabel.nil?
+
+          @alabel = alabel
+          @ulabel = ulabel
+
+          super(ulabel || alabel, **options)
+        end
+
+        def call
+          alabels, alabel_trailing_dot = split_domain(alabel.encode("UTF-8").unicode_normalize(:nfc)) if alabel
+          ulabels, ulabel_trailing_dot = split_domain(ulabel.encode("UTF-8").unicode_normalize(:nfc)) if ulabel
+
+          if alabels && ulabels && (alabels.size != ulabels.size || alabel_trailing_dot != ulabel_trailing_dot)
+            raise Error, "alabel doesn't match ulabel"
+          end
+
+          trailing_dot = alabel_trailing_dot || ulabel_trailing_dot
+          size = (alabels || ulabels).size
+
+          result = Array.new(size) do |i|
+            alabel = alabels&.[](i)
+            ulabel = ulabels&.[](i)
+
+            raise Error, "Provided alabel must be downcased" if alabel && alabel.downcase != alabel
+
+            if alabel
+              u_alabel = punycode_decode(alabel)
+              if ulabel && u_alabel != ulabel
+                raise Error,
+                      "Provided ulabel #{ulabel.inspect} doesn't match punycoded alabel #{u_alabel.inspect}"
+              end
+            end
+
+            validate(ulabel || punycode_decode(alabel))
+            a_ulabel = punycode_encode(ulabel || punycode_decode(alabel))
+
+            Validation::Label.check_length(a_ulabel) if options.verify_dns_length?
+
+            if alabel && ulabel && (a_ulabel != alabel) && (a_ulabel != alabel)
+              raise Error,
+                    "Provided alabel #{alabel.inspect} doesn't match de-punycoded ulabel #{u_alabel.inspect}"
+            end
+
+            a_ulabel
+          end
+
+          result = join_labels(result, trailing_dot)
+
+          Validation::Label.check_domain_length(result) if options.verify_dns_length?
+          result
+        end
+
+        private
+
+        attr_reader :ulabel, :alabel
+
+        def validate(label)
+          Validation::Label.check_hyphen_sides(label) if options.check_hyphens?
+          super
+        end
+      end
+
+      # # https://datatracker.ietf.org/doc/html/rfc5891#section-5
+      class Lookup < Processing
+        def call
+          domain = domain_name.encode("UTF-8").unicode_normalize(:nfc)
+
+          result = process_labels(domain) do |label|
+            orig_label = label
+            alabel_input = label.start_with?(ACE_PREFIX)
+
+            label = punycode_decode(label)
+
+            validate(label)
+
+            label = punycode_encode(label)
+
+            Validation::Label.check_length(label) if options.verify_dns_length?
+
+            if alabel_input && orig_label != label
+              raise Error, "Resulting label #{label.inspect} doesn't match initial label #{orig_label.inspect}"
+            end
+
+            label
+          end
+
+          Validation::Label.check_domain_length(result) if options.verify_dns_length?
+          result
+        end
+      end
+    end
+  end
+end

data/lib/uri/idna/intranges.rb

@@ -23,12 +23,14 @@ module URI
           # we could be immediately ahead of a tuple (start, end)
           # with start < int_ <= end
           if pos > 0
-            left, right = decode_range(ranges[pos - 1])
+            r = ranges[pos - 1]
+            left = decode_left(r)
+            right = decode_right(r)
             return true if left <= int && int < right
           end
           # or we could be immediately behind a tuple (int_, end)
           if pos < ranges.length
-            left, = decode_range(ranges[pos])
+            left = decode_left(ranges[pos])
             return true if left == int
           end
           false
@@ -40,8 +42,14 @@ module URI
           (start << 32) | finish
         end
 
-        def decode_range(r)
-          [(r >> 32), (r & ((1 << 32) - 1))]
+        MASK = ((1 << 32) - 1)
+
+        def decode_right(r)
+          r & MASK
+        end
+
+        def decode_left(r)
+          r >> 32
         end
       end
     end

data/lib/uri/idna/punycode.rb

@@ -47,20 +47,21 @@ module URI
 
         def encode(input)
           input = input.codepoints
+          output = []
 
           n = INITIAL_N
           delta = 0
           bias = INITIAL_BIAS
 
-          output = input.select { |char| basic?(char) }
+          input.each { |cp| output << cp if cp < 0x80 }
           h = b = output.length
 
           output << DELIMITER if b > 0
 
           while h < input.length
             m = MAXINT
-            input.each do |char|
-              m = char if char >= n && char < m
+            input.each do |cp|
+              m = cp if cp >= n && cp < m
             end
 
             raise PunycodeError, "Arithmetic overflow" if m - n > (MAXINT - delta) / (h + 1)
@@ -68,12 +69,12 @@ module URI
             delta += (m - n) * (h + 1)
             n = m
 
-            input.each do |char|
-              if char < n
+            input.each do |cp|
+              if cp < n
                 delta += 1
                 raise PunycodeError, "Arithmetic overflow" if delta > MAXINT
               end
-              next unless char == n
+              next unless cp == n
 
               q = delta
               k = BASE
@@ -115,10 +116,11 @@ module URI
 
           b = input.rindex(DELIMITER) || 0
 
-          input[0, b].each do |char|
-            raise PunycodeError, "Invalid input" unless basic?(char)
+          0.upto(b - 1) do |idx|
+            cp = input[idx]
+            raise PunycodeError, "Invalid input" unless cp < 0x80
 
-            output << char
+            output << cp
           end
 
           inc = b > 0 ? b + 1 : 0
@@ -162,12 +164,6 @@ module URI
 
           output.pack("U*")
         end
-
-        private
-
-        def basic?(codepoint)
-          codepoint < 0x80
-        end
       end
     end
   end

data/lib/uri/idna/uts46/mapping.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require_relative "../data/uts46"
+
+module URI
+  module IDNA
+    module UTS46
+      # https://www.unicode.org/reports/tr46/#IDNA_Mapping_Table
+      module Mapping
+        class << self
+          def call(domain_name, transitional_processing: false, use_std3_ascii_rules: true)
+            output = +""
+            domain_name.each_codepoint do |codepoint|
+              _, status, replacement = status(codepoint)
+              case status
+              when "V", "X" # valid, disallowed
+                output << codepoint.chr(Encoding::UTF_8)
+              when "M" # mapped
+                output << if transitional_processing && codepoint == 7838
+                            "ss"
+                          else
+                            replacement
+                          end
+              when "D" # deviation
+                output << (transitional_processing ? replacement : codepoint.chr(Encoding::UTF_8))
+              when "3" # disallowed_STD3_valid, disallowed_STD3_mapped
+                output << if use_std3_ascii_rules
+                            codepoint.chr(Encoding::UTF_8)
+                          else
+                            (replacement || codepoint.chr(Encoding::UTF_8))
+                          end
+              when "I" # ignored
+                next
+              end
+            end
+            output.unicode_normalize(:nfc)
+          end
+
+          def validate_status(label, cp, pos, transitional_processing:, use_std3_ascii_rules:)
+            _, status, = status(cp)
+            return if status == "V"
+            return if !transitional_processing && status == "D"
+            return if !use_std3_ascii_rules && status == "3"
+
+            raise InvalidCodepointError, Validation::Codepoint.cp_error_message(label, cp, pos)
+          end
+
+          def status(codepoint)
+            index =
+              if codepoint < 256
+                codepoint
+              else
+                (UTS46_DATA.bsearch_index { |x| x[0] > codepoint } || UTS46_DATA.length) - 1
+              end
+            UTS46_DATA[index] || []
+          end
+        end
+      end
+    end
+  end
+end

data/lib/uri/idna/uts46/options.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module URI
+  module IDNA
+    module UTS46
+      # Options for ToUnicode operation
+      # https://www.unicode.org/reports/tr46/#ToUnicode
+      class Options
+        USE_STD3_ASCII_RULES      = 1 << 0
+        CHECK_HYPHENS             = 1 << 1
+        CHECK_BIDI                = 1 << 2
+        CHECK_JOINERS             = 1 << 3
+        TRANSITIONAL_PROCESSING   = 1 << 4
+        IGNORE_INVALID_PUNYCODE   = 1 << 5
+
+        def initialize(
+          use_std3_ascii_rules: true,
+          check_hyphens: true,
+          check_bidi: true,
+          check_joiners: true,
+          transitional_processing: false,
+          ignore_invalid_punycode: false
+        )
+          @flags = 0
+          @flags |= USE_STD3_ASCII_RULES if use_std3_ascii_rules
+          @flags |= CHECK_HYPHENS if check_hyphens
+          @flags |= CHECK_BIDI if check_bidi
+          @flags |= CHECK_JOINERS if check_joiners
+          @flags |= TRANSITIONAL_PROCESSING if transitional_processing
+          @flags |= IGNORE_INVALID_PUNYCODE if ignore_invalid_punycode
+        end
+
+        def use_std3_ascii_rules?
+          (@flags & USE_STD3_ASCII_RULES) != 0
+        end
+
+        def check_hyphens?
+          (@flags & CHECK_HYPHENS) != 0
+        end
+
+        def check_bidi?
+          (@flags & CHECK_BIDI) != 0
+        end
+
+        def check_joiners?
+          (@flags & CHECK_JOINERS) != 0
+        end
+
+        def transitional_processing?
+          (@flags & TRANSITIONAL_PROCESSING) != 0
+        end
+
+        def ignore_invalid_punycode?
+          (@flags & IGNORE_INVALID_PUNYCODE) != 0
+        end
+      end
+
+      # Options for ToASCII operation
+      # https://www.unicode.org/reports/tr46/#ToASCII
+      class ToASCIIOptions < Options
+        VERIFY_DNS_LENGTH = 1 << 6
+
+        def initialize(verify_dns_length: true, **options)
+          @flags_extended = 0
+          @flags_extended |= VERIFY_DNS_LENGTH if verify_dns_length
+          super(**options)
+        end
+
+        def verify_dns_length?
+          (@flags_extended & VERIFY_DNS_LENGTH) != 0
+        end
+      end
+    end
+  end
+end

data/lib/uri/idna/uts46/processing.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require_relative "mapping"
+require_relative "options"
+
+module URI
+  module IDNA
+    module UTS46
+      # https://www.unicode.org/reports/tr46/#Processing
+      class Processing < BaseProcessing
+        def call
+          domain = Mapping.call(
+            domain_name,
+            transitional_processing: options.transitional_processing?,
+            use_std3_ascii_rules: options.use_std3_ascii_rules?,
+          )
+
+          process_labels(domain) do |label|
+            if label.start_with?(ACE_PREFIX)
+              begin
+                label = punycode_decode(label)
+              rescue PunycodeError => e
+                next label if options.ignore_invalid_punycode?
+
+                raise e
+              end
+              validate(label, transitional_processing: false)
+            else
+              validate(label)
+            end
+
+            label = yield label if block_given?
+
+            label
+          end
+        end
+
+        private
+
+        def options_class
+          Options
+        end
+
+        def check_bidi?
+          return @check_bidi if instance_variable_defined?(:@check_bidi)
+
+          @check_bidi = options.check_bidi? && Validation::Bidi.check?(domain_name)
+        end
+
+        # https://www.unicode.org/reports/tr46/#Validity_Criteria
+        def validate(label, transitional_processing: options.transitional_processing?)
+          return if label.empty?
+
+          Validation::Label.check_nfc(label)
+          if options.check_hyphens?
+            Validation::Label.check_hyphen34(label)
+            Validation::Label.check_hyphen_sides(label)
+          else
+            Validation::Label.check_ace_prefix(label)
+          end
+          Validation::Label.check_dot(label)
+          Validation::Label.check_leading_combining(label)
+
+          label.each_codepoint.with_index do |cp, pos|
+            Mapping.validate_status(
+              label, cp, pos,
+              transitional_processing: transitional_processing, use_std3_ascii_rules: options.use_std3_ascii_rules?
+            )
+
+            Validation::Codepoint.check_contextj(label, cp, pos) if options.check_joiners?
+          end
+          Validation::Bidi.call(label) if check_bidi?
+        end
+      end
+
+      # https://www.unicode.org/reports/tr46/#ToUnicode
+      class ToUnicode < Processing
+      end
+
+      # https://www.unicode.org/reports/tr46/#ToASCII
+      class ToASCII < Processing
+        def options_class
+          ToASCIIOptions
+        end
+
+        def call
+          result = super do |label|
+            label = punycode_encode(label)
+            Validation::Label.check_length(label) if options.verify_dns_length?
+            label
+          end
+          Validation::Label.check_domain_length(result) if options.verify_dns_length?
+          result
+        end
+      end
+    end
+  end
+end

data/lib/uri/idna/validation/bidi.rb

@@ -2,7 +2,7 @@
 
 module URI
   module IDNA
-    class Validation
+    module Validation
       # 4.2.3.4. Labels Containing Characters Written Right to Left
       # https://datatracker.ietf.org/doc/html/rfc5891#section-4.2.3.4
       # https://datatracker.ietf.org/doc/html/rfc5893#section-2
@@ -10,9 +10,9 @@ module URI
         class << self
           def call(label)
             # Bidi rule 1
-            if bidi_class(label[0], "RTL")
+            if bidi_class(label[0].ord, "RTL")
               rtl = true
-            elsif bidi_class(label[0], "L")
+            elsif bidi_class(label[0].ord, "L")
               rtl = false
             else
               raise BidiError, "First codepoint in label #{label} must be directionality L, R or AL"
@@ -20,11 +20,11 @@ module URI
 
             valid_ending = false
             number_type = nil
-            label.each_char.with_index do |cp, idx|
+            label.each_codepoint.with_index do |cp, pos|
               if rtl
                 # Bidi rule 2
                 if bidi_class(cp, "L") || bidi_class(cp, "UNUSED")
-                  raise BidiError, "Invalid direction for codepoint at position #{idx + 1} in a right-to-left label"
+                  raise BidiError, "Invalid direction for codepoint at position #{pos + 1} in a right-to-left label"
                 end
 
                 # Bidi rule 3
@@ -42,7 +42,7 @@ module URI
               else
                 # Bidi rule 5
                 if bidi_class(cp, "RTL") || bidi_class(cp, "AN")
-                  raise BidiError, "Invalid direction for codepoint at position #{idx + 1} in a left-to-right label"
+                  raise BidiError, "Invalid direction for codepoint at position #{pos + 1} in a left-to-right label"
                 end
 
                 # Bidi rule 6
@@ -60,12 +60,13 @@ module URI
           end
 
           # https://www.rfc-editor.org/rfc/rfc5891.html#section-4.2.3.4
-          def check?(labels)
+          def check?(domain)
+            labels = domain.split(".", -1)
             domain = labels.map do |label|
-              if label.start_with?(ALABEL_PREFIX)
+              if label.start_with?(ACE_PREFIX)
                 begin
-                  Punycode.decode(label[ALABEL_PREFIX.length..])
-                rescue StandardError
+                  Punycode.decode(label[ACE_PREFIX.length..])
+                rescue PunycodeError
                   ""
                 end
               else
@@ -73,7 +74,7 @@ module URI
               end
             end.join(".")
 
-            domain.each_char do |cp|
+            domain.each_codepoint do |cp|
               return true if bidi_class(cp, "RTL") || bidi_class(cp, "AN")
             end
             false
@@ -81,8 +82,8 @@ module URI
 
           private
 
-          def bidi_class(cp, bidi_class)
-            return bidi_class if Intranges.contain?(cp.ord, BIDI_CLASSES[bidi_class])
+          def bidi_class(codepoint, bidi_class)
+            return bidi_class if Intranges.contain?(codepoint, BIDI_CLASSES[bidi_class])
 
             false
           end

data/lib/uri/idna/validation/codepoint.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require_relative "../intranges"
+require_relative "../data/idna"
+
+module URI
+  module IDNA
+    module Validation
+      module Codepoint
+        class << self
+          # https://datatracker.ietf.org/doc/html/rfc5891#section-4.2.2
+          def check_idna_validity(label, cp, pos)
+            return true if codepoint?(cp, "PVALID")
+
+            raise InvalidCodepointError, cp_error_message(label, cp, pos)
+          end
+
+          # https://datatracker.ietf.org/doc/html/rfc5892
+          def check_contextj(label, cp, pos)
+            return false unless codepoint?(cp, "CONTEXTJ")
+            return true if valid_contextj?(label, cp, pos)
+
+            raise InvalidCodepointContextError, cp_error_message(label, cp, pos)
+          end
+
+          # https://datatracker.ietf.org/doc/html/rfc5892
+          def check_contexto(label, cp, pos)
+            return false unless codepoint?(cp, "CONTEXTO")
+            return true if valid_contexto?(label, cp, pos)
+
+            raise InvalidCodepointContextError, cp_error_message(label, cp, pos)
+          end
+
+          def cp_error_message(label, cp, pos)
+            format("Codepoint U+%04X at position %d of %p not allowed", cp, pos + 1, label)
+          end
+
+          private
+
+          def valid_contexto?(label, cp, pos)
+            case cp
+              # https://datatracker.ietf.org/doc/html/rfc5892#appendix-A.3
+            when 0x00b7
+              pos > 0 && pos < label.length - 1 ? (label[pos - 1].ord == 0x006c && label[pos + 1].ord == 0x006c) : false
+              # https://datatracker.ietf.org/doc/html/rfc5892#appendix-A.4
+            when 0x0375
+              pos < label.length - 1 ? script?(label[pos + 1].ord, "Greek") : false
+              # https://datatracker.ietf.org/doc/html/rfc5892#appendix-A.5
+              # https://datatracker.ietf.org/doc/html/rfc5892#appendix-A.6
+            when 0x05f3, 0x05f4
+              pos > 0 ? script?(label[pos - 1].ord, "Hebrew") : false
+              # https://datatracker.ietf.org/doc/html/rfc5892#appendix-A.7
+            when 0x30fb
+              label.each_codepoint do |codepoint|
+                next if codepoint == 0x30fb
+                return true if script?(codepoint,
+                                       "Hiragana") || script?(codepoint, "Katakana") || script?(codepoint, "Han")
+              end
+              false
+              # https://datatracker.ietf.org/doc/html/rfc5892#appendix-A.8
+            when 0x0660..0x0669
+              label.each_codepoint do |codepoint|
+                return false if codepoint >= 0x06f0 && codepoint <= 0x06f9
+              end
+              true
+              # https://datatracker.ietf.org/doc/html/rfc5892#appendix-A.9
+            when 0x06f0..0x06f9
+              label.each_codepoint do |codepoint|
+                return false if codepoint >= 0x0660 && codepoint <= 0x0669
+              end
+              true
+            else
+              false
+            end
+          end
+
+          def valid_contextj?(label, cp, pos)
+            case cp
+              # https://datatracker.ietf.org/doc/html/rfc5892#appendix-A.1
+            when 0x200c
+              return true if pos > 0 && virama_combining_class?(label[pos - 1].ord)
+
+              ok = false
+              (pos - 1).downto(0) do |i|
+                joining_type = JOINING_TYPES[label[i].ord]
+                next if joining_type == 0x54
+
+                if [0x4c, 0x44].include?(joining_type)
+                  ok = true
+                  break
+                end
+              end
+              return false unless ok
+
+              (pos + 1).upto(label.length - 1) do |i|
+                joining_type = JOINING_TYPES[label[i].ord]
+                next if joining_type == 0x54
+                return true if [0x52, 0x44].include?(joining_type)
+              end
+              # https://datatracker.ietf.org/doc/html/rfc5892#appendix-A.2
+            when 0x200d
+              return virama_combining_class?(label[pos - 1].ord) if pos > 0
+            end
+            false
+          end
+
+          def script?(cp, script)
+            Intranges.contain?(cp, SCRIPTS[script])
+          end
+
+          def virama_combining_class?(cp)
+            Intranges.contain?(cp, VIRAMA_COMBINING_CLASSES)
+          end
+
+          def codepoint?(cp, class_name)
+            Intranges.contain?(cp, CODEPOINT_CLASSES[class_name])
+          end
+        end
+      end
+    end
+  end
+end