kebab 1.0.2

data/lib/kebab/utf8/active_support_proxy.rb

@@ -0,0 +1,26 @@
+require 'active_support'
+require 'active_support/multibyte/unicode'
+
+module Kebab
+  module UTF8
+    # A UTF-8 proxy using Active Support's multibyte support.
+    module ActiveSupportProxy
+      extend ActiveSupport::Multibyte::Unicode
+      extend self
+
+      def self.normalize_utf8(string)
+        normalize(string, :c)
+      end
+
+      if ActiveSupport::VERSION::MAJOR == 3
+        def downcase(string)
+          ActiveSupport::Multibyte::Chars.new(string).downcase.to_s
+        end
+
+        def upcase(string)
+          ActiveSupport::Multibyte::Chars.new(string).upcase.to_s
+        end
+      end
+    end
+  end
+end

data/lib/kebab/utf8/dumb_proxy.rb

@@ -0,0 +1,49 @@
+require File.expand_path("../mappings", __FILE__)
+
+module Kebab
+  module UTF8
+
+    # This module provides fallback UTF-8 support when nothing else is
+    # available. It does case folding for Roman alphabet-based characters
+    # commonly used by Western European languages and little else, making it
+    # useless for Russian, Bulgarian, Greek, etc. If at all possible, Unicode
+    # or ActiveSupport should be used instead because they support the full
+    # UTF-8 character range.
+    module DumbProxy
+      extend Proxy
+      extend self
+
+      def downcase(string)
+        string.downcase.unpack("U*").map {|char| Mappings::DOWNCASE[char] or char}.flatten.pack("U*")
+      end
+
+      def upcase(string)
+        string.upcase.unpack("U*").map {|char| Mappings::UPCASE[char] or char}.flatten.pack("U*")
+      end
+
+      if ''.respond_to?(:unicode_normalize)
+        def normalize_utf8(string)
+          string.unicode_normalize
+        end
+      else
+        # On Ruby 2.2, this uses the native Unicode normalize method. On all
+        # other Rubies, it does a very naive Unicode normalization, which should
+        # work for this library's purposes (i.e., Roman-based codepoints, up to
+        # U+017E).  Do not use reuse this as a general solution!  Use a real
+        # library like Unicode or ActiveSupport instead.
+        def normalize_utf8(string)
+          codepoints = string.unpack("U*")
+          new = []
+          until codepoints.empty? do
+            if Mappings::COMPOSITION[codepoints[0..1]]
+              new << Mappings::COMPOSITION[codepoints.slice!(0,2)]
+            else
+              new << codepoints.shift
+            end
+          end
+          new.compact.flatten.pack("U*")
+        end
+      end
+    end
+  end
+end

data/lib/kebab/utf8/java_proxy.rb

@@ -0,0 +1,22 @@
+module Kebab
+  module UTF8
+    # A UTF-8 proxy module using Java's built-in Unicode support. Requires JRuby 1.5+.
+    module JavaProxy
+      extend Proxy
+      extend self
+      java_import java.text.Normalizer
+
+      def downcase(string)
+        string.to_java.to_lower_case.to_s
+      end
+
+      def upcase(string)
+        string.to_java.to_upper_case.to_s
+      end
+
+      def normalize_utf8(string)
+        Normalizer.normalize(string, Normalizer::Form::NFC).to_s
+      end
+    end
+  end
+end

data/lib/kebab/utf8/mappings.rb

@@ -0,0 +1,193 @@
+module Kebab
+  module UTF8
+
+    # A small subset of the mappings provided by Unicode.org, limited to Latin
+    # characters. This is used for Kebab's default "dumb" UTF-8 support.
+    module Mappings
+      DOWNCASE = Hash[65, 97, 66, 98, 67, 99, 68, 100, 69, 101, 70, 102,
+        71, 103, 72, 104, 73, 105, 74, 106, 75, 107, 76, 108, 77, 109, 78, 110,
+        79, 111, 80, 112, 81, 113, 82, 114, 83, 115, 84, 116, 85, 117, 86, 118,
+        87, 119, 88, 120, 89, 121, 90, 122, 181, 956, 192, 224, 193, 225, 194,
+        226, 195, 227, 196, 228, 197, 229, 198, 230, 199, 231, 200, 232, 201,
+        233, 202, 234, 203, 235, 204, 236, 205, 237, 206, 238, 207, 239, 208,
+        240, 209, 241, 210, 242, 211, 243, 212, 244, 213, 245, 214, 246, 216,
+        248, 217, 249, 218, 250, 219, 251, 220, 252, 221, 253, 222, 254, 223,
+        [115, 115], 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267,
+        268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281,
+        282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295,
+        296, 297, 298, 299, 300, 301, 302, 303, 304, [105, 775], 306, 307, 308,
+        309, 310, 311, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323,
+        324, 325, 326, 327, 328, 329, [700, 110], 330, 331, 332, 333, 334, 335,
+        336, 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349,
+        350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363,
+        364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 255,
+        377, 378, 379, 380, 381, 382]
+
+      UPCASE = DOWNCASE.invert
+
+      COMPOSITION = {
+        [65,768]  => 192,
+        [65,769]  => 193,
+        [65,770]  => 194,
+        [65,771]  => 195,
+        [65,776]  => 196,
+        [65,778]  => 197,
+        [67,807]  => 199,
+        [69,768]  => 200,
+        [69,769]  => 201,
+        [69,770]  => 202,
+        [69,776]  => 203,
+        [73,768]  => 204,
+        [73,769]  => 205,
+        [73,770]  => 206,
+        [73,776]  => 207,
+        [78,771]  => 209,
+        [79,768]  => 210,
+        [79,769]  => 211,
+        [79,770]  => 212,
+        [79,771]  => 213,
+        [79,776]  => 214,
+        [85,768]  => 217,
+        [85,769]  => 218,
+        [85,770]  => 219,
+        [85,776]  => 220,
+        [89,769]  => 221,
+        [97,768]  => 224,
+        [97,769]  => 225,
+        [97,770]  => 226,
+        [97,771]  => 227,
+        [97,776]  => 228,
+        [97,778]  => 229,
+        [99,807]  => 231,
+        [101,768] => 232,
+        [101,769] => 233,
+        [101,770] => 234,
+        [101,776] => 235,
+        [105,768] => 236,
+        [105,769] => 237,
+        [105,770] => 238,
+        [105,776] => 239,
+        [110,771] => 241,
+        [111,768] => 242,
+        [111,769] => 243,
+        [111,770] => 244,
+        [111,771] => 245,
+        [111,776] => 246,
+        [117,768] => 249,
+        [117,769] => 250,
+        [117,770] => 251,
+        [117,776] => 252,
+        [121,769] => 253,
+        [121,776] => 255,
+        [65,772]  => 256,
+        [97,772]  => 257,
+        [65,774]  => 258,
+        [97,774]  => 259,
+        [65,808]  => 260,
+        [97,808]  => 261,
+        [67,769]  => 262,
+        [99,769]  => 263,
+        [67,770]  => 264,
+        [99,770]  => 265,
+        [67,775]  => 266,
+        [99,775]  => 267,
+        [67,780]  => 268,
+        [99,780]  => 269,
+        [68,780]  => 270,
+        [100,780] => 271,
+        [69,772]  => 274,
+        [101,772] => 275,
+        [69,774]  => 276,
+        [101,774] => 277,
+        [69,775]  => 278,
+        [101,775] => 279,
+        [69,808]  => 280,
+        [101,808] => 281,
+        [69,780]  => 282,
+        [101,780] => 283,
+        [71,770]  => 284,
+        [103,770] => 285,
+        [71,774]  => 286,
+        [103,774] => 287,
+        [71,775]  => 288,
+        [103,775] => 289,
+        [71,807]  => 290,
+        [103,807] => 291,
+        [72,770]  => 292,
+        [104,770] => 293,
+        [73,771]  => 296,
+        [105,771] => 297,
+        [73,772]  => 298,
+        [105,772] => 299,
+        [73,774]  => 300,
+        [105,774] => 301,
+        [73,808]  => 302,
+        [105,808] => 303,
+        [73,775]  => 304,
+        [74,770]  => 308,
+        [106,770] => 309,
+        [75,807]  => 310,
+        [107,807] => 311,
+        [76,769]  => 313,
+        [108,769] => 314,
+        [76,807]  => 315,
+        [108,807] => 316,
+        [76,780]  => 317,
+        [108,780] => 318,
+        [78,769]  => 323,
+        [110,769] => 324,
+        [78,807]  => 325,
+        [110,807] => 326,
+        [78,780]  => 327,
+        [110,780] => 328,
+        [79,772]  => 332,
+        [111,772] => 333,
+        [79,774]  => 334,
+        [111,774] => 335,
+        [79,779]  => 336,
+        [111,779] => 337,
+        [82,769]  => 340,
+        [114,769] => 341,
+        [82,807]  => 342,
+        [114,807] => 343,
+        [82,780]  => 344,
+        [114,780] => 345,
+        [83,769]  => 346,
+        [115,769] => 347,
+        [83,770]  => 348,
+        [115,770] => 349,
+        [83,807]  => 350,
+        [115,807] => 351,
+        [83,780]  => 352,
+        [115,780] => 353,
+        [84,807]  => 354,
+        [116,807] => 355,
+        [84,780]  => 356,
+        [116,780] => 357,
+        [85,771]  => 360,
+        [117,771] => 361,
+        [85,772]  => 362,
+        [117,772] => 363,
+        [85,774]  => 364,
+        [117,774] => 365,
+        [85,778]  => 366,
+        [117,778] => 367,
+        [85,779]  => 368,
+        [117,779] => 369,
+        [85,808]  => 370,
+        [117,808] => 371,
+        [87,770]  => 372,
+        [119,770] => 373,
+        [89,770]  => 374,
+        [121,770] => 375,
+        [89,776]  => 376,
+        [90,769]  => 377,
+        [122,769] => 378,
+        [90,775]  => 379,
+        [122,775] => 380,
+        [90,780]  => 381,
+        [122,780] => 382
+      }
+    end
+  end
+end

data/lib/kebab/utf8/proxy.rb

@@ -0,0 +1,125 @@
+module Kebab
+  module UTF8
+
+    autoload :JavaProxy,          "kebab/utf8/java_proxy"
+    autoload :UnicodeProxy,       "kebab/utf8/unicode_proxy"
+    autoload :ActiveSupportProxy, "kebab/utf8/active_support_proxy"
+    autoload :DumbProxy,          "kebab/utf8/dumb_proxy"
+
+    # A UTF-8 proxy for Kebab can be any object which responds to the methods in this module.
+    # The following proxies are provided by Kebab: {ActiveSupportProxy}, {DumbProxy}, {JavaProxy}, and {UnicodeProxy}.
+    module Proxy
+      CP1252  = {
+        128 => [226, 130, 172],
+        129 => nil,
+        130 => [226, 128, 154],
+        131 => [198, 146],
+        132 => [226, 128, 158],
+        133 => [226, 128, 166],
+        134 => [226, 128, 160],
+        135 => [226, 128, 161],
+        136 => [203, 134],
+        137 => [226, 128, 176],
+        138 => [197, 160],
+        139 => [226, 128, 185],
+        140 => [197, 146],
+        141 => nil,
+        142 => [197, 189],
+        143 => nil,
+        144 => nil,
+        145 => [226, 128, 152],
+        146 => [226, 128, 153],
+        147 => [226, 128, 156],
+        148 => [226, 128, 157],
+        149 => [226, 128, 162],
+        150 => [226, 128, 147],
+        151 => [226, 128, 148],
+        152 => [203, 156],
+        153 => [226, 132, 162],
+        154 => [197, 161],
+        155 => [226, 128, 186],
+        156 => [197, 147],
+        157 => nil,
+        158 => [197, 190],
+        159 => [197, 184]
+      }
+
+      # This is a stub for a method that should return a Unicode-aware
+      # downcased version of the given string.
+      def downcase(string)
+        raise NotImplementedError
+      end
+
+      # This is a stub for a method that should return a Unicode-aware
+      # upcased version of the given string.
+      def upcase(string)
+        raise NotImplementedError
+      end
+
+      # This is a stub for a method that should return the Unicode NFC
+      # normalization of the given string.
+      def normalize_utf8(string)
+        raise NotImplementedError
+      end
+
+      if ''.respond_to?(:scrub) && !defined?(Rubinius)
+        # Attempt to replace invalid UTF-8 bytes with valid ones. This method
+        # naively assumes if you have invalid UTF8 bytes, they are either Windows
+        # CP-1252 or ISO8859-1. In practice this isn't a bad assumption, but may not
+        # always work.
+        def tidy_bytes(string)
+          string.scrub do |bad|
+            tidy_byte(*bad.bytes).flatten.compact.pack('C*').unpack('U*').pack('U*')
+          end
+        end
+      else
+        def tidy_bytes(string)
+          bytes = string.unpack("C*")
+          conts_expected = 0
+          last_lead = 0
+
+          bytes.each_index do |i|
+            byte          = bytes[i]
+            is_cont       = byte > 127 && byte < 192
+            is_lead       = byte > 191 && byte < 245
+            is_unused     = byte > 240
+            is_restricted = byte > 244
+
+            # Impossible or highly unlikely byte? Clean it.
+            if is_unused || is_restricted
+              bytes[i] = tidy_byte(byte)
+            elsif is_cont
+              # Not expecting contination byte? Clean up. Otherwise, now expect one less.
+              conts_expected == 0 ? bytes[i] = tidy_byte(byte) : conts_expected -= 1
+            else
+              if conts_expected > 0
+                # Expected continuation, but got ASCII or leading? Clean backwards up to
+                # the leading byte.
+                (1..(i - last_lead)).each {|j| bytes[i - j] = tidy_byte(bytes[i - j])}
+                conts_expected = 0
+              end
+              if is_lead
+                # Final byte is leading? Clean it.
+                if i == bytes.length - 1
+                  bytes[i] = tidy_byte(bytes.last)
+                else
+                  # Valid leading byte? Expect continuations determined by position of
+                  # first zero bit, with max of 3.
+                  conts_expected = byte < 224 ? 1 : byte < 240 ? 2 : 3
+                  last_lead = i
+                end
+              end
+            end
+          end
+          bytes.empty? ? "" : bytes.flatten.compact.pack("C*").unpack("U*").pack("U*")
+        end
+      end
+
+      private
+
+      def tidy_byte(byte)
+        byte < 160 ? CP1252[byte] : byte < 192 ? [194, byte] : [195, byte - 64]
+      end
+    end
+  end
+end

data/lib/kebab/utf8/unicode_proxy.rb

@@ -0,0 +1,23 @@
+require 'unicode'
+
+module Kebab
+  module UTF8
+    # A UTF-8 proxy using the Unicode gem.
+    # @see http://github.com/blackwinter/unicode
+    module UnicodeProxy
+      extend Proxy
+      extend self
+      def downcase(string)
+        Unicode.downcase(string)
+      end
+
+      def upcase(string)
+        Unicode.upcase(string)
+      end
+
+      def normalize_utf8(string)
+        Unicode.normalize_C(string)
+      end
+    end
+  end
+end

data/lib/kebab/version.rb

@@ -0,0 +1,5 @@
+module Kebab
+  module Version
+    STRING = '1.0.2'
+  end
+end