babosa 1.0.4 → 2.0.0

data/lib/babosa/utf8/dumb_proxy.rb

@@ -1,49 +0,0 @@
-require File.expand_path("../mappings", __FILE__)
-
-module Babosa
-  module UTF8
-
-    # This module provides fallback UTF-8 support when nothing else is
-    # available. It does case folding for Roman alphabet-based characters
-    # commonly used by Western European languages and little else, making it
-    # useless for Russian, Bulgarian, Greek, etc. If at all possible, Unicode
-    # or ActiveSupport should be used instead because they support the full
-    # UTF-8 character range.
-    module DumbProxy
-      extend Proxy
-      extend self
-
-      def downcase(string)
-        string.downcase.unpack("U*").map {|char| Mappings::DOWNCASE[char] or char}.flatten.pack("U*")
-      end
-
-      def upcase(string)
-        string.upcase.unpack("U*").map {|char| Mappings::UPCASE[char] or char}.flatten.pack("U*")
-      end
-
-      if ''.respond_to?(:unicode_normalize)
-        def normalize_utf8(string)
-          string.unicode_normalize
-        end
-      else
-        # On Ruby 2.2, this uses the native Unicode normalize method. On all
-        # other Rubies, it does a very naive Unicode normalization, which should
-        # work for this library's purposes (i.e., Roman-based codepoints, up to
-        # U+017E).  Do not use reuse this as a general solution!  Use a real
-        # library like Unicode or ActiveSupport instead.
-        def normalize_utf8(string)
-          codepoints = string.unpack("U*")
-          new = []
-          until codepoints.empty? do
-            if Mappings::COMPOSITION[codepoints[0..1]]
-              new << Mappings::COMPOSITION[codepoints.slice!(0,2)]
-            else
-              new << codepoints.shift
-            end
-          end
-          new.compact.flatten.pack("U*")
-        end
-      end
-    end
-  end
-end

data/lib/babosa/utf8/java_proxy.rb

@@ -1,22 +0,0 @@
-module Babosa
-  module UTF8
-    # A UTF-8 proxy module using Java's built-in Unicode support. Requires JRuby 1.5+.
-    module JavaProxy
-      extend Proxy
-      extend self
-      java_import java.text.Normalizer
-
-      def downcase(string)
-        string.to_java.to_lower_case.to_s
-      end
-
-      def upcase(string)
-        string.to_java.to_upper_case.to_s
-      end
-
-      def normalize_utf8(string)
-        Normalizer.normalize(string, Normalizer::Form::NFC).to_s
-      end
-    end
-  end
-end

data/lib/babosa/utf8/mappings.rb

@@ -1,193 +0,0 @@
-module Babosa
-  module UTF8
-
-    # A small subset of the mappings provided by Unicode.org, limited to Latin
-    # characters. This is used for Babosa's default "dumb" UTF-8 support.
-    module Mappings
-      DOWNCASE = Hash[65, 97, 66, 98, 67, 99, 68, 100, 69, 101, 70, 102,
-        71, 103, 72, 104, 73, 105, 74, 106, 75, 107, 76, 108, 77, 109, 78, 110,
-        79, 111, 80, 112, 81, 113, 82, 114, 83, 115, 84, 116, 85, 117, 86, 118,
-        87, 119, 88, 120, 89, 121, 90, 122, 181, 956, 192, 224, 193, 225, 194,
-        226, 195, 227, 196, 228, 197, 229, 198, 230, 199, 231, 200, 232, 201,
-        233, 202, 234, 203, 235, 204, 236, 205, 237, 206, 238, 207, 239, 208,
-        240, 209, 241, 210, 242, 211, 243, 212, 244, 213, 245, 214, 246, 216,
-        248, 217, 249, 218, 250, 219, 251, 220, 252, 221, 253, 222, 254, 223,
-        [115, 115], 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267,
-        268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281,
-        282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295,
-        296, 297, 298, 299, 300, 301, 302, 303, 304, [105, 775], 306, 307, 308,
-        309, 310, 311, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323,
-        324, 325, 326, 327, 328, 329, [700, 110], 330, 331, 332, 333, 334, 335,
-        336, 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349,
-        350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363,
-        364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 255,
-        377, 378, 379, 380, 381, 382]
-
-      UPCASE = DOWNCASE.invert
-
-      COMPOSITION = {
-        [65,768]  => 192,
-        [65,769]  => 193,
-        [65,770]  => 194,
-        [65,771]  => 195,
-        [65,776]  => 196,
-        [65,778]  => 197,
-        [67,807]  => 199,
-        [69,768]  => 200,
-        [69,769]  => 201,
-        [69,770]  => 202,
-        [69,776]  => 203,
-        [73,768]  => 204,
-        [73,769]  => 205,
-        [73,770]  => 206,
-        [73,776]  => 207,
-        [78,771]  => 209,
-        [79,768]  => 210,
-        [79,769]  => 211,
-        [79,770]  => 212,
-        [79,771]  => 213,
-        [79,776]  => 214,
-        [85,768]  => 217,
-        [85,769]  => 218,
-        [85,770]  => 219,
-        [85,776]  => 220,
-        [89,769]  => 221,
-        [97,768]  => 224,
-        [97,769]  => 225,
-        [97,770]  => 226,
-        [97,771]  => 227,
-        [97,776]  => 228,
-        [97,778]  => 229,
-        [99,807]  => 231,
-        [101,768] => 232,
-        [101,769] => 233,
-        [101,770] => 234,
-        [101,776] => 235,
-        [105,768] => 236,
-        [105,769] => 237,
-        [105,770] => 238,
-        [105,776] => 239,
-        [110,771] => 241,
-        [111,768] => 242,
-        [111,769] => 243,
-        [111,770] => 244,
-        [111,771] => 245,
-        [111,776] => 246,
-        [117,768] => 249,
-        [117,769] => 250,
-        [117,770] => 251,
-        [117,776] => 252,
-        [121,769] => 253,
-        [121,776] => 255,
-        [65,772]  => 256,
-        [97,772]  => 257,
-        [65,774]  => 258,
-        [97,774]  => 259,
-        [65,808]  => 260,
-        [97,808]  => 261,
-        [67,769]  => 262,
-        [99,769]  => 263,
-        [67,770]  => 264,
-        [99,770]  => 265,
-        [67,775]  => 266,
-        [99,775]  => 267,
-        [67,780]  => 268,
-        [99,780]  => 269,
-        [68,780]  => 270,
-        [100,780] => 271,
-        [69,772]  => 274,
-        [101,772] => 275,
-        [69,774]  => 276,
-        [101,774] => 277,
-        [69,775]  => 278,
-        [101,775] => 279,
-        [69,808]  => 280,
-        [101,808] => 281,
-        [69,780]  => 282,
-        [101,780] => 283,
-        [71,770]  => 284,
-        [103,770] => 285,
-        [71,774]  => 286,
-        [103,774] => 287,
-        [71,775]  => 288,
-        [103,775] => 289,
-        [71,807]  => 290,
-        [103,807] => 291,
-        [72,770]  => 292,
-        [104,770] => 293,
-        [73,771]  => 296,
-        [105,771] => 297,
-        [73,772]  => 298,
-        [105,772] => 299,
-        [73,774]  => 300,
-        [105,774] => 301,
-        [73,808]  => 302,
-        [105,808] => 303,
-        [73,775]  => 304,
-        [74,770]  => 308,
-        [106,770] => 309,
-        [75,807]  => 310,
-        [107,807] => 311,
-        [76,769]  => 313,
-        [108,769] => 314,
-        [76,807]  => 315,
-        [108,807] => 316,
-        [76,780]  => 317,
-        [108,780] => 318,
-        [78,769]  => 323,
-        [110,769] => 324,
-        [78,807]  => 325,
-        [110,807] => 326,
-        [78,780]  => 327,
-        [110,780] => 328,
-        [79,772]  => 332,
-        [111,772] => 333,
-        [79,774]  => 334,
-        [111,774] => 335,
-        [79,779]  => 336,
-        [111,779] => 337,
-        [82,769]  => 340,
-        [114,769] => 341,
-        [82,807]  => 342,
-        [114,807] => 343,
-        [82,780]  => 344,
-        [114,780] => 345,
-        [83,769]  => 346,
-        [115,769] => 347,
-        [83,770]  => 348,
-        [115,770] => 349,
-        [83,807]  => 350,
-        [115,807] => 351,
-        [83,780]  => 352,
-        [115,780] => 353,
-        [84,807]  => 354,
-        [116,807] => 355,
-        [84,780]  => 356,
-        [116,780] => 357,
-        [85,771]  => 360,
-        [117,771] => 361,
-        [85,772]  => 362,
-        [117,772] => 363,
-        [85,774]  => 364,
-        [117,774] => 365,
-        [85,778]  => 366,
-        [117,778] => 367,
-        [85,779]  => 368,
-        [117,779] => 369,
-        [85,808]  => 370,
-        [117,808] => 371,
-        [87,770]  => 372,
-        [119,770] => 373,
-        [89,770]  => 374,
-        [121,770] => 375,
-        [89,776]  => 376,
-        [90,769]  => 377,
-        [122,769] => 378,
-        [90,775]  => 379,
-        [122,775] => 380,
-        [90,780]  => 381,
-        [122,780] => 382
-      }
-    end
-  end
-end

data/lib/babosa/utf8/proxy.rb

@@ -1,125 +0,0 @@
-module Babosa
-  module UTF8
-
-    autoload :JavaProxy,          "babosa/utf8/java_proxy"
-    autoload :UnicodeProxy,       "babosa/utf8/unicode_proxy"
-    autoload :ActiveSupportProxy, "babosa/utf8/active_support_proxy"
-    autoload :DumbProxy,          "babosa/utf8/dumb_proxy"
-
-    # A UTF-8 proxy for Babosa can be any object which responds to the methods in this module.
-    # The following proxies are provided by Babosa: {ActiveSupportProxy}, {DumbProxy}, {JavaProxy}, and {UnicodeProxy}.
-    module Proxy
-      CP1252  = {
-        128 => [226, 130, 172],
-        129 => nil,
-        130 => [226, 128, 154],
-        131 => [198, 146],
-        132 => [226, 128, 158],
-        133 => [226, 128, 166],
-        134 => [226, 128, 160],
-        135 => [226, 128, 161],
-        136 => [203, 134],
-        137 => [226, 128, 176],
-        138 => [197, 160],
-        139 => [226, 128, 185],
-        140 => [197, 146],
-        141 => nil,
-        142 => [197, 189],
-        143 => nil,
-        144 => nil,
-        145 => [226, 128, 152],
-        146 => [226, 128, 153],
-        147 => [226, 128, 156],
-        148 => [226, 128, 157],
-        149 => [226, 128, 162],
-        150 => [226, 128, 147],
-        151 => [226, 128, 148],
-        152 => [203, 156],
-        153 => [226, 132, 162],
-        154 => [197, 161],
-        155 => [226, 128, 186],
-        156 => [197, 147],
-        157 => nil,
-        158 => [197, 190],
-        159 => [197, 184]
-      }
-
-      # This is a stub for a method that should return a Unicode-aware
-      # downcased version of the given string.
-      def downcase(string)
-        raise NotImplementedError
-      end
-
-      # This is a stub for a method that should return a Unicode-aware
-      # upcased version of the given string.
-      def upcase(string)
-        raise NotImplementedError
-      end
-
-      # This is a stub for a method that should return the Unicode NFC
-      # normalization of the given string.
-      def normalize_utf8(string)
-        raise NotImplementedError
-      end
-
-      if ''.respond_to?(:scrub) && !defined?(Rubinius)
-        # Attempt to replace invalid UTF-8 bytes with valid ones. This method
-        # naively assumes if you have invalid UTF8 bytes, they are either Windows
-        # CP-1252 or ISO8859-1. In practice this isn't a bad assumption, but may not
-        # always work.
-        def tidy_bytes(string)
-          string.scrub do |bad|
-            tidy_byte(*bad.bytes).flatten.compact.pack('C*').unpack('U*').pack('U*')
-          end
-        end
-      else
-        def tidy_bytes(string)
-          bytes = string.unpack("C*")
-          conts_expected = 0
-          last_lead = 0
-
-          bytes.each_index do |i|
-            byte          = bytes[i]
-            is_cont       = byte > 127 && byte < 192
-            is_lead       = byte > 191 && byte < 245
-            is_unused     = byte > 240
-            is_restricted = byte > 244
-
-            # Impossible or highly unlikely byte? Clean it.
-            if is_unused || is_restricted
-              bytes[i] = tidy_byte(byte)
-            elsif is_cont
-              # Not expecting contination byte? Clean up. Otherwise, now expect one less.
-              conts_expected == 0 ? bytes[i] = tidy_byte(byte) : conts_expected -= 1
-            else
-              if conts_expected > 0
-                # Expected continuation, but got ASCII or leading? Clean backwards up to
-                # the leading byte.
-                (1..(i - last_lead)).each {|j| bytes[i - j] = tidy_byte(bytes[i - j])}
-                conts_expected = 0
-              end
-              if is_lead
-                # Final byte is leading? Clean it.
-                if i == bytes.length - 1
-                  bytes[i] = tidy_byte(bytes.last)
-                else
-                  # Valid leading byte? Expect continuations determined by position of
-                  # first zero bit, with max of 3.
-                  conts_expected = byte < 224 ? 1 : byte < 240 ? 2 : 3
-                  last_lead = i
-                end
-              end
-            end
-          end
-          bytes.empty? ? "" : bytes.flatten.compact.pack("C*").unpack("U*").pack("U*")
-        end
-      end
-
-      private
-
-      def tidy_byte(byte)
-        byte < 160 ? CP1252[byte] : byte < 192 ? [194, byte] : [195, byte - 64]
-      end
-    end
-  end
-end

data/lib/babosa/utf8/unicode_proxy.rb

@@ -1,23 +0,0 @@
-require 'unicode'
-
-module Babosa
-  module UTF8
-    # A UTF-8 proxy using the Unicode gem.
-    # @see http://github.com/blackwinter/unicode
-    module UnicodeProxy
-      extend Proxy
-      extend self
-      def downcase(string)
-        Unicode.downcase(string)
-      end
-
-      def upcase(string)
-        Unicode.upcase(string)
-      end
-
-      def normalize_utf8(string)
-        Unicode.normalize_C(string)
-      end
-    end
-  end
-end

data/spec/utf8_proxy_spec.rb

@@ -1,52 +0,0 @@
-# encoding: utf-8
-require File.expand_path("../spec_helper", __FILE__)
-
-PROXIES = [Babosa::UTF8::DumbProxy, Babosa::UTF8::ActiveSupportProxy, Babosa::UTF8::UnicodeProxy]
-PROXIES << Babosa::UTF8::JavaProxy if Babosa.jruby15?
-
-PROXIES.each do |proxy|
-
-  describe proxy do
-
-    around do |example|
-      begin
-        old_proxy = Babosa::Identifier.utf8_proxy
-        Babosa::Identifier.utf8_proxy = proxy
-        example.run
-      ensure
-        Babosa::Identifier.utf8_proxy = old_proxy
-      end
-    end
-
-    describe "#normalize_utf8" do
-      it "should normalize to canonical composed" do
-        # ÅÉÎØÜ
-        uncomposed_bytes  = [65, 204, 138, 69, 204, 129, 73, 204, 130, 195, 152, 85, 204, 136]
-        composed_bytes    = [195, 133, 195, 137, 195, 142, 195, 152, 195, 156]
-        uncomposed_string = uncomposed_bytes.pack("C*").unpack("U*").pack("U*")
-        expect(proxy.normalize_utf8(uncomposed_string).unpack("C*")).to eql(composed_bytes)
-      end
-    end
-
-    describe "#upcase" do
-      it "should upcase the string" do
-        expect(proxy.upcase("åéîøü")).to eql("ÅÉÎØÜ")
-        expect("åéîøü".to_identifier.upcase).to eql("ÅÉÎØÜ")
-      end
-    end
-
-    describe "#downcase" do
-      it "should downcase the string" do
-        expect(proxy.downcase("ÅÉÎØÜ")).to eql("åéîøü")
-        expect("ÅÉÎØÜ".to_identifier.downcase).to eql("åéîøü")
-      end
-    end
-
-    describe 'tidy_bytes' do
-      it 'should fix invalid UTF-8 strings' do
-        expect(proxy.tidy_bytes("\x93abc")).to eq('“abc')
-      end
-    end
-
-  end
-end