babosa 0.3.10 → 1.0.4

data/lib/babosa/transliterator/macedonian.rb

@@ -12,6 +12,7 @@ module Babosa
         "Ц" => "C",
         "Ѕ" => "Z",
         "Ј" => "J",
+        "Х" => "H",
         "ѓ" => "gj",
         "љ" => "lj",
         "њ" => "nj",
@@ -20,7 +21,8 @@ module Babosa
         "ж" => "zh",
         "ц" => "c",
         "ѕ" => "z",
-        "ј" => "j"
+        "ј" => "j",
+        "х" => "h"
       }
     end
   end

data/lib/babosa/transliterator/turkish.rb

@@ -0,0 +1,8 @@
+# encoding: utf-8
+
+module Babosa
+  module Transliterator
+    class Turkish < Latin
+    end
+  end
+end

data/lib/babosa/transliterator/ukrainian.rb

@@ -3,8 +3,27 @@ module Babosa
   module Transliterator
     class Ukrainian < Cyrillic
       APPROXIMATIONS = {
+        "Г" => "H",
+        "г" => "h",
+        "Ґ" => "G",
+        "ґ" => "g",
+        "є" => "ie",
         "И" => "Y",
         "и" => "y",
+        "І" => "I",
+        "і" => "i",
+        "ї" => "i",
+        "Й" => "Y",
+        "й" => "i",
+        "Х" => "Kh",
+        "х" => "kh",
+        "Ц" => "Ts",
+        "ц" => 'ts',
+        "Щ" => "Shch",
+        "щ" => "shch",
+        "ю" => "iu",
+        "я" => "ia",
+        "'" => ""
       }
     end
   end

data/lib/babosa/transliterator/vietnamese.rb

@@ -0,0 +1,143 @@
+# encoding: utf-8
+module Babosa
+  module Transliterator
+    class Vietnamese < Latin
+      APPROXIMATIONS = {
+        "à" => "a",
+        "á" => "a",
+        "ạ" => "a",
+        "ả" => "a",
+        "ã" => "a",
+        "â" => "a",
+        "ầ" => "a",
+        "ấ" => "a",
+        "ậ" => "a",
+        "ẩ" => "a",
+        "ẫ" => "a",
+        "ă" => "a",
+        "ằ" => "a",
+        "ắ" => "a",
+        "ặ" => "a",
+        "ẳ" => "a",
+        "ẵ" => "a",
+        "À" => "A",
+        "Á" => "A",
+        "Ạ" => "A",
+        "Ả" => "A",
+        "Ã" => "A",
+        "Â" => "A",
+        "Ầ" => "A",
+        "Ấ" => "A",
+        "Ậ" => "A",
+        "Ẩ" => "A",
+        "Ẫ" => "A",
+        "Ă" => "A",
+        "Ằ" => "A",
+        "Ắ" => "A",
+        "Ặ" => "A",
+        "Ẳ" => "A",
+        "Ẵ" => "A",
+        "ì" => "i",
+        "í" => "i",
+        "ị" => "i",
+        "ỉ" => "i",
+        "ĩ" => "i",
+        "Ì" => "I",
+        "Í" => "I",
+        "Ị" => "I",
+        "Ỉ" => "I",
+        "Ĩ" => "I",
+        "ù" => "u",
+        "ú" => "u",
+        "ụ" => "u",
+        "ủ" => "u",
+        "ũ" => "u",
+        "ư" => "u",
+        "ừ" => "u",
+        "ứ" => "u",
+        "ự" => "u",
+        "ử" => "u",
+        "ữ" => "u",
+        "Ù" => "U",
+        "Ú" => "U",
+        "Ụ" => "U",
+        "Ủ" => "U",
+        "Ũ" => "U",
+        "Ư" => "U",
+        "Ừ" => "U",
+        "Ứ" => "U",
+        "Ự" => "U",
+        "Ử" => "U",
+        "Ữ" => "U",
+        "è" => "e",
+        "é" => "e",
+        "ẹ" => "e",
+        "ẻ" => "e",
+        "ẽ" => "e",
+        "ê" => "e",
+        "ề" => "e",
+        "ế" => "e",
+        "ệ" => "e",
+        "ể" => "e",
+        "ễ" => "e",
+        "È" => "E",
+        "É" => "E",
+        "Ẹ" => "E",
+        "Ẻ" => "E",
+        "Ẽ" => "E",
+        "Ê" => "E",
+        "Ề" => "E",
+        "Ế" => "E",
+        "Ệ" => "E",
+        "Ể" => "E",
+        "Ễ" => "E",
+        "ò" => "o",
+        "ó" => "o",
+        "ọ" => "o",
+        "ỏ" => "o",
+        "õ" => "o",
+        "ô" => "o",
+        "ồ" => "o",
+        "ố" => "o",
+        "ộ" => "o",
+        "ổ" => "o",
+        "ỗ" => "o",
+        "ơ" => "o",
+        "ờ" => "o",
+        "ớ" => "o",
+        "ợ" => "o",
+        "ở" => "o",
+        "ỡ" => "o",
+        "Ò" => "O",
+        "Ó" => "O",
+        "Ọ" => "O",
+        "Ỏ" => "O",
+        "Õ" => "O",
+        "Ô" => "O",
+        "Ồ" => "O",
+        "Ố" => "O",
+        "Ộ" => "O",
+        "Ổ" => "O",
+        "Ỗ" => "O",
+        "Ơ" => "O",
+        "Ờ" => "O",
+        "Ớ" => "O",
+        "Ợ" => "O",
+        "Ở" => "O",
+        "Ỡ" => "O",
+        "ỳ" => "y",
+        "ý" => "y",
+        "ỵ" => "y",
+        "ỷ" => "y",
+        "ỹ" => "y",
+        "Ỳ" => "Y",
+        "Ý" => "Y",
+        "Ỵ" => "Y",
+        "Ỷ" => "Y",
+        "Ỹ" => "Y",
+        "đ" => "d",
+        "Đ" => "D"
+      }
+    end
+  end
+end

data/lib/babosa/utf8/active_support_proxy.rb

@@ -1,19 +1,37 @@
+require 'active_support'
+require 'active_support/multibyte/unicode'
+
 module Babosa
   module UTF8
     # A UTF-8 proxy using Active Support's multibyte support.
     module ActiveSupportProxy
-      extend UTF8Proxy
+      extend ActiveSupport::Multibyte::Unicode
       extend self
-      def downcase(string)
-        ActiveSupport::Multibyte::Chars.new(string).downcase.to_s
-      end
 
-      def upcase(string)
-        ActiveSupport::Multibyte::Chars.new(string).upcase.to_s
+      def self.normalize_utf8(string)
+        normalize(string, :c)
       end
 
-      def normalize_utf8(string)
-        ActiveSupport::Multibyte::Chars.new(string).normalize(:c).to_s
+      if ActiveSupport::VERSION::MAJOR == 3
+        def downcase(string)
+          ActiveSupport::Multibyte::Chars.new(string).downcase.to_s
+        end
+
+        def upcase(string)
+          ActiveSupport::Multibyte::Chars.new(string).upcase.to_s
+        end
+      elsif ActiveSupport::VERSION::MAJOR >= 6
+        def self.normalize_utf8(string)
+          string.unicode_normalize(:nfc).to_s
+        end
+
+        def downcase(string)
+          string.downcase.to_s
+        end
+
+        def upcase(string)
+          string.upcase.to_s
+        end
       end
     end
   end

data/lib/babosa/utf8/dumb_proxy.rb

@@ -10,32 +10,39 @@ module Babosa
     # or ActiveSupport should be used instead because they support the full
     # UTF-8 character range.
     module DumbProxy
-      extend UTF8Proxy
+      extend Proxy
       extend self
 
       def downcase(string)
-        string.unpack("U*").map {|char| Mappings::DOWNCASE[char] or char}.flatten.pack("U*")
+        string.downcase.unpack("U*").map {|char| Mappings::DOWNCASE[char] or char}.flatten.pack("U*")
       end
 
       def upcase(string)
-        string.unpack("U*").map {|char| Mappings::UPCASE[char] or char}.flatten.pack("U*")
+        string.upcase.unpack("U*").map {|char| Mappings::UPCASE[char] or char}.flatten.pack("U*")
       end
 
-      # This does a very naive Unicode normalization, which should work for
-      # this library's purposes (i.e., Roman-based codepoints, up to U+017E).
-      # Do not use reuse this as a general solution!  Use a real library like
-      # Unicode or ActiveSupport instead.
-      def normalize_utf8(string)
-        codepoints = string.unpack("U*")
-        new = []
-        until codepoints.empty? do
-          if Mappings::COMPOSITION[codepoints[0..1]]
-            new << Mappings::COMPOSITION[codepoints.slice!(0,2)]
-          else
-            new << codepoints.shift
+      if ''.respond_to?(:unicode_normalize)
+        def normalize_utf8(string)
+          string.unicode_normalize
+        end
+      else
+        # On Ruby 2.2, this uses the native Unicode normalize method. On all
+        # other Rubies, it does a very naive Unicode normalization, which should
+        # work for this library's purposes (i.e., Roman-based codepoints, up to
+        # U+017E).  Do not use reuse this as a general solution!  Use a real
+        # library like Unicode or ActiveSupport instead.
+        def normalize_utf8(string)
+          codepoints = string.unpack("U*")
+          new = []
+          until codepoints.empty? do
+            if Mappings::COMPOSITION[codepoints[0..1]]
+              new << Mappings::COMPOSITION[codepoints.slice!(0,2)]
+            else
+              new << codepoints.shift
+            end
           end
+          new.compact.flatten.pack("U*")
         end
-        new.compact.flatten.pack("U*")
       end
     end
   end

data/lib/babosa/utf8/java_proxy.rb

@@ -2,7 +2,7 @@ module Babosa
   module UTF8
     # A UTF-8 proxy module using Java's built-in Unicode support. Requires JRuby 1.5+.
     module JavaProxy
-      extend UTF8Proxy
+      extend Proxy
       extend self
       java_import java.text.Normalizer
 

data/lib/babosa/utf8/proxy.rb

@@ -8,7 +8,7 @@ module Babosa
 
     # A UTF-8 proxy for Babosa can be any object which responds to the methods in this module.
     # The following proxies are provided by Babosa: {ActiveSupportProxy}, {DumbProxy}, {JavaProxy}, and {UnicodeProxy}.
-    module UTF8Proxy
+    module Proxy
       CP1252  = {
         128 => [226, 130, 172],
         129 => nil,
@@ -62,50 +62,57 @@ module Babosa
         raise NotImplementedError
       end
 
-      # Attempt to replace invalid UTF-8 bytes with valid ones. This method
-      # naively assumes if you have invalid UTF8 bytes, they are either Windows
-      # CP-1252 or ISO8859-1. In practice this isn't a bad assumption, but may not
-      # always work.
-      def tidy_bytes(string)
-        bytes = string.unpack("C*")
-        conts_expected = 0
-        last_lead = 0
+      if ''.respond_to?(:scrub) && !defined?(Rubinius)
+        # Attempt to replace invalid UTF-8 bytes with valid ones. This method
+        # naively assumes if you have invalid UTF8 bytes, they are either Windows
+        # CP-1252 or ISO8859-1. In practice this isn't a bad assumption, but may not
+        # always work.
+        def tidy_bytes(string)
+          string.scrub do |bad|
+            tidy_byte(*bad.bytes).flatten.compact.pack('C*').unpack('U*').pack('U*')
+          end
+        end
+      else
+        def tidy_bytes(string)
+          bytes = string.unpack("C*")
+          conts_expected = 0
+          last_lead = 0
 
-        bytes.each_index do |i|
-          byte          = bytes[i]
-          is_ascii      = byte < 128
-          is_cont       = byte > 127 && byte < 192
-          is_lead       = byte > 191 && byte < 245
-          is_unused     = byte > 240
-          is_restricted = byte > 244
+          bytes.each_index do |i|
+            byte          = bytes[i]
+            is_cont       = byte > 127 && byte < 192
+            is_lead       = byte > 191 && byte < 245
+            is_unused     = byte > 240
+            is_restricted = byte > 244
 
-          # Impossible or highly unlikely byte? Clean it.
-          if is_unused || is_restricted
-            bytes[i] = tidy_byte(byte)
-          elsif is_cont
-            # Not expecting contination byte? Clean up. Otherwise, now expect one less.
-            conts_expected == 0 ? bytes[i] = tidy_byte(byte) : conts_expected -= 1
-          else
-            if conts_expected > 0
-              # Expected continuation, but got ASCII or leading? Clean backwards up to
-              # the leading byte.
-              (1..(i - last_lead)).each {|j| bytes[i - j] = tidy_byte(bytes[i - j])}
-              conts_expected = 0
-            end
-            if is_lead
-              # Final byte is leading? Clean it.
-              if i == bytes.length - 1
-                bytes[i] = tidy_byte(bytes.last)
-              else
-                # Valid leading byte? Expect continuations determined by position of
-                # first zero bit, with max of 3.
-                conts_expected = byte < 224 ? 1 : byte < 240 ? 2 : 3
-                last_lead = i
+            # Impossible or highly unlikely byte? Clean it.
+            if is_unused || is_restricted
+              bytes[i] = tidy_byte(byte)
+            elsif is_cont
+              # Not expecting contination byte? Clean up. Otherwise, now expect one less.
+              conts_expected == 0 ? bytes[i] = tidy_byte(byte) : conts_expected -= 1
+            else
+              if conts_expected > 0
+                # Expected continuation, but got ASCII or leading? Clean backwards up to
+                # the leading byte.
+                (1..(i - last_lead)).each {|j| bytes[i - j] = tidy_byte(bytes[i - j])}
+                conts_expected = 0
+              end
+              if is_lead
+                # Final byte is leading? Clean it.
+                if i == bytes.length - 1
+                  bytes[i] = tidy_byte(bytes.last)
+                else
+                  # Valid leading byte? Expect continuations determined by position of
+                  # first zero bit, with max of 3.
+                  conts_expected = byte < 224 ? 1 : byte < 240 ? 2 : 3
+                  last_lead = i
+                end
               end
             end
           end
+          bytes.empty? ? "" : bytes.flatten.compact.pack("C*").unpack("U*").pack("U*")
         end
-        bytes.empty? ? "" : bytes.flatten.compact.pack("C*").unpack("U*").pack("U*")
       end
 
       private

data/lib/babosa/utf8/unicode_proxy.rb

@@ -1,9 +1,11 @@
+require 'unicode'
+
 module Babosa
   module UTF8
     # A UTF-8 proxy using the Unicode gem.
     # @see http://github.com/blackwinter/unicode
     module UnicodeProxy
-      extend UTF8Proxy
+      extend Proxy
       extend self
       def downcase(string)
         Unicode.downcase(string)

data/lib/babosa/version.rb

@@ -1,5 +1,5 @@
 module Babosa
   module Version
-    STRING = "0.3.10"
+    STRING = '1.0.4'
   end
 end

data/spec/babosa_spec.rb

@@ -4,7 +4,7 @@ require File.expand_path("../spec_helper", __FILE__)
 describe Babosa::Identifier do
 
   it "should respond_to :empty?" do
-    "".to_slug.should respond_to(:empty?)
+    expect("".to_slug).to respond_to(:empty?)
   end
 
   %w[approximate_ascii clean downcase word_chars normalize to_ascii upcase with_dashes].each do |method|
@@ -18,128 +18,141 @@ describe Babosa::Identifier do
   describe "#word_chars" do
     it "word_chars! should leave only letters and spaces" do
       string = "a*$%^$@!@b$%^&*()*!c"
-      string.to_slug.word_chars.should match(/[a-z ]*/i)
+      expect(string.to_slug.word_chars!).to match(/[a-z ]*/i)
     end
   end
 
   describe "#transliterate" do
     it "should transliterate to ascii" do
-      slug = (0xC0..0x17E).to_a.each do |codepoint|
+      (0xC0..0x17E).to_a.each do |codepoint|
         ss = [codepoint].pack("U*").to_slug
-        ss.approximate_ascii.should match(/[\x0-\x7f]/)
+        expect(ss.approximate_ascii!).to match(/[\x0-\x7f]/)
       end
     end
 
     it "should transliterate uncomposed utf8" do
       string = [117, 776].pack("U*") # "ü" as ASCII "u" plus COMBINING DIAERESIS
-      string.to_slug.approximate_ascii.should eql("u")
+      expect(string.to_slug.approximate_ascii).to eql("u")
     end
 
     it "should transliterate using multiple transliterators" do
       string = "свободное režģis"
-      string.to_slug.approximate_ascii(:latin, :russian).should eql("svobodnoe rezgis")
+      expect(string.to_slug.approximate_ascii(:latin, :russian)).to eql("svobodnoe rezgis")
     end
   end
 
   describe "#downcase" do
     it "should lowercase strings" do
-      "FELIZ AÑO".to_slug.downcase.should eql("feliz año")
+      expect("FELIZ AÑO".to_slug.downcase).to eql("feliz año")
     end
   end
 
   describe "#upcase" do
     it "should uppercase strings" do
-      "feliz año".to_slug.upcase.should eql("FELIZ AÑO")
+      expect("feliz año".to_slug.upcase).to eql("FELIZ AÑO")
     end
   end
 
   describe "#normalize" do
 
     it "should allow passing locale as key for :transliterate" do
-      "ö".to_slug.clean.normalize(:transliterate => :german).should eql("oe")
+      expect("ö".to_slug.clean.normalize(:transliterate => :german)).to eql("oe")
     end
 
     it "should replace whitespace with dashes" do
-      "a b".to_slug.clean.normalize.should eql("a-b")
+      expect("a b".to_slug.clean.normalize).to eql("a-b")
     end
 
     it "should replace multiple spaces with 1 dash" do
-      "a    b".to_slug.clean.normalize.should eql("a-b")
+      expect("a    b".to_slug.clean.normalize).to eql("a-b")
     end
 
     it "should replace multiple dashes with 1 dash" do
-      "male - female".to_slug.normalize.should eql("male-female")
+      expect("male - female".to_slug.normalize).to eql("male-female")
     end
 
     it "should strip trailing space" do
-      "ab ".to_slug.normalize.should eql("ab")
+      expect("ab ".to_slug.normalize).to eql("ab")
     end
 
     it "should strip leading space" do
-      " ab".to_slug.normalize.should eql("ab")
+      expect(" ab".to_slug.normalize).to eql("ab")
     end
 
     it "should strip trailing slashes" do
-      "ab-".to_slug.normalize.should eql("ab")
+      expect("ab-".to_slug.normalize).to eql("ab")
     end
 
     it "should strip leading slashes" do
-      "-ab".to_slug.normalize.should eql("ab")
+      expect("-ab".to_slug.normalize).to eql("ab")
     end
 
     it "should not modify valid name strings" do
-      "a-b-c-d".to_slug.normalize.should eql("a-b-c-d")
+      expect("a-b-c-d".to_slug.normalize).to eql("a-b-c-d")
     end
 
     it "should not convert underscores" do
-      "hello_world".to_slug.normalize.should eql("hello_world")
+      expect("hello_world".to_slug.normalize).to eql("hello_world")
     end
 
     it "should work with non roman chars" do
-      "検 索".to_slug.normalize.should eql("検-索")
+      expect("検 索".to_slug.normalize).to eql("検-索")
     end
 
     context "with to_ascii option" do
       it "should approximate and strip non ascii" do
         ss = "カタカナ: katakana is über cool".to_slug
-        ss.normalize(:to_ascii => true).should eql("katakana-is-uber-cool")
+        expect(ss.normalize(:to_ascii => true)).to eql("katakana-is-uber-cool")
       end
     end
   end
 
   describe "#truncate_bytes" do
     it "should by byte length" do
-      "üa".to_slug.truncate_bytes(2).should eql("ü")
-      "üa".to_slug.truncate_bytes(1).should eql("")
-      "üa".to_slug.truncate_bytes(100).should eql("üa")
-      "üéøá".to_slug.truncate_bytes(3).should eql("ü")
+      expect("üa".to_slug.truncate_bytes(2)).to eql("ü")
+      expect("üa".to_slug.truncate_bytes(1)).to eql("")
+      expect("üa".to_slug.truncate_bytes(100)).to eql("üa")
+      expect("üéøá".to_slug.truncate_bytes(3)).to eql("ü")
     end
   end
 
   describe "#truncate" do
     it "should truncate by char length" do
-      "üa".to_slug.truncate(2).should eql("üa")
-      "üa".to_slug.truncate(1).should eql("ü")
-      "üa".to_slug.truncate(100).should eql("üa")
+      expect("üa".to_slug.truncate(2)).to eql("üa")
+      expect("üa".to_slug.truncate(1)).to eql("ü")
+      expect("üa".to_slug.truncate(100)).to eql("üa")
     end
   end
 
   describe "#with_dashes" do
     it "should not change byte size when replacing spaces" do
-      "".to_slug.with_dashes.bytesize.should eql(0)
-      " ".to_slug.with_dashes.bytesize.should eql(1)
-      "-abc-".to_slug.with_dashes.bytesize.should eql(5)
-      " abc ".to_slug.with_dashes.bytesize.should eql(5)
-      " a  bc ".to_slug.with_dashes.bytesize.should eql(7)
+      expect("".to_slug.with_dashes.bytesize).to eql(0)
+      expect(" ".to_slug.with_dashes.bytesize).to eql(1)
+      expect("-abc-".to_slug.with_dashes.bytesize).to eql(5)
+      expect(" abc ".to_slug.with_dashes.bytesize).to eql(5)
+      expect(" a  bc ".to_slug.with_dashes.bytesize).to eql(7)
     end
   end
 
   describe "#to_ruby_method" do
     it "should get a string suitable for use as a ruby method" do
-      "¿¿¿hello... world???".to_slug.to_ruby_method.should eql("hello_world?")
-      "カタカナ: katakana is über cool".to_slug.to_ruby_method.should eql("katakana_is_uber_cool")
-      "カタカナ: katakana is über cool!".to_slug.to_ruby_method.should eql("katakana_is_uber_cool!")
-      "カタカナ: katakana is über cool".to_slug.to_ruby_method(false).should eql("katakana_is_uber_cool")
+      expect("¿¿¿hello... world???".to_slug.to_ruby_method).to eql("hello_world?")
+      expect("カタカナ: katakana is über cool".to_slug.to_ruby_method).to eql("katakana_is_uber_cool")
+      expect("カタカナ: katakana is über cool!".to_slug.to_ruby_method).to eql("katakana_is_uber_cool!")
+      expect("カタカナ: katakana is über cool".to_slug.to_ruby_method(false)).to eql("katakana_is_uber_cool")
+    end
+
+    it "should optionally remove trailing punctuation" do
+      expect("¿¿¿hello... world???".to_slug.to_ruby_method(false)).to eql("hello_world")
+    end
+
+    it "should raise an error when it would generate an impossible method name" do
+      # "1".to_identifier.to_ruby_method
+      expect {"1".to_identifier.to_ruby_method}.to raise_error(Babosa::Identifier::Error)
+    end
+
+    it "should raise Babosa::Error error when the string is nil" do
+      expect { "".to_slug.to_ruby_method }.to raise_error(Babosa::Identifier::Error)
     end
   end
-end
+end