babosa 0.1.0

data/MIT-LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2010 Norman Clarke
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,91 @@
+# Babosa
+
+Babosa is a library for creating slugs. It is an extraction and improvement of
+the string code from [FriendlyId](http://github.com/norman/friendly_id),
+intended to help developers create libraries similar to FriendlyId.
+
+## Features / Usage
+
+### ASCII transliteration
+
+    "Gölcük, Turkey".to_slug.approximate_ascii.to_s #=> "Golcuk, Turkey"
+
+### Special cases for German and Spanish
+
+    "Jürgen Müller".to_slug.approximate_ascii.to_s           #=> "Jurgen Muller"
+    "Jürgen Müller".to_slug.approximate_ascii(:german).to_s  #=> "Juergen Mueller"
+    "feliz año".to_slug.approximate_ascii.to_s               #=> "feliz ano"
+    "feliz año".to_slug.approximate_ascii(:spanish).to_s     #=> "feliz anio"
+
+### Non-ASCII removal
+
+    "Gölcük, Turkey".to_slug.to_ascii.to_s #=> "Glck, Turkey"
+
+### Truncate by characters
+
+    "üüü".to_slug.truncate(2).to_s #=> "üü"
+
+### Truncate by bytes
+
+This can be useful to ensure the generated slug will fit in a database column
+whose length is limited by bytes rather than UTF-8 characters.
+
+    "üüü".to_slug.truncate_bytes(2).to_s #=> "ü"
+
+### All-in-one
+
+    "Gölcük, Turkey".to_slug.normalize.to_s #=> "golcuk-turkey"
+
+There are many more features; check the API docs and source code to find out
+more.
+
+## Getting it
+
+Babosa can be installed via Rubygems:
+
+    gem install babosa
+
+You can get the source code from its [Github repository](http://github.com/norman/babosa).
+
+## Reporting bugs
+
+Please use Babosa's [Github issue tracker](http://github.com/norman/babosa/issues).
+
+
+## Misc
+
+The speed and quality of Babosa's UTF-8 support depends on which Ruby and which
+gems you are using.
+
+On JRuby 1.5 and above, Babosa uses Java's native UTF-8 support. If you require
+[Unicode](http://github.com/blackwinter/unicode) or ActiveSupport before
+Babosa, it will use the support provided by those libraries. Otherwise, Babosa
+defaults to very basic UTF-8 support for Latin characters only.
+
+"Babosa" means slug in Spanish.
+
+## Author
+
+[Norman Clarke](http://njclarke.com)
+
+## Copyright
+
+Copyright (c) 2010 Norman Clarke
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/Rakefile

@@ -0,0 +1,27 @@
+require "rake/testtask"
+require "rake/clean"
+require "rake/gempackagetask"
+
+task :default => :test
+
+CLEAN << "pkg" << "doc" << "coverage" << ".yardoc"
+Rake::GemPackageTask.new(eval(File.read("babosa.gemspec"))) { |pkg| }
+Rake::TestTask.new(:test) { |t| t.pattern = "test/**/*_test.rb" }
+
+begin
+  require "yard"
+  YARD::Rake::YardocTask.new do |t|
+    t.options = ["--output-dir=doc"]
+  end
+rescue LoadError
+end
+
+begin
+  require "rcov/rcovtask"
+  Rcov::RcovTask.new do |r|
+    r.test_files = FileList["test/**/*_test.rb"]
+    r.verbose = true
+    r.rcov_opts << "--exclude gems/*"
+  end
+rescue LoadError
+end

data/init.rb

@@ -0,0 +1,3 @@
+$LOAD_PATH << File.expand_path("../lib", __FILE__)
+$LOAD_PATH.uniq!
+require "babosa"

data/lib/babosa.rb

@@ -0,0 +1,22 @@
+module Babosa
+  def self.jruby15?
+    JRUBY_VERSION >= "1.5" rescue false
+  end
+end
+
+class String
+  def to_slug
+    Babosa::SlugString.new self
+  end
+
+  # Compatibility with 1.8.6
+  if !public_method_defined? :bytesize
+    def bytesize
+      unpack("C*").length
+    end
+  end
+end
+
+require "babosa/characters"
+require "babosa/utf8/proxy"
+require "babosa/slug_string"

data/lib/babosa/characters.rb

@@ -0,0 +1,75 @@
+# encoding: utf-8
+module Babosa
+
+  # This module provides sets of characters needed for various UTF-8 aware
+  # string operations.
+  module Characters
+    extend self
+
+    # Hash of UTF-8 - ASCII approximations.
+    attr_reader :approximations
+    # Punctuation and control characters to remove from slug strings.
+    attr_reader :strippable
+
+    @strippable = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 14, 15, 16, 17, 18, 19,
+      20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 33, 34, 35, 36, 37, 38, 39,
+      40, 41, 42, 43, 44, 45, 46, 47, 58, 59, 60, 61, 62, 63, 64, 91, 92, 93, 94,
+      95, 96, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135,
+      136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150,
+      151, 152, 153, 154, 155, 156, 157, 158, 159, 161, 162, 163, 164, 165, 166,
+      167, 168, 169, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 182, 183,
+      184, 185, 187, 188, 189, 190, 191, 215, 247]
+
+    # Adds a hash of approximations.
+    # @example
+    #     add_approximations :spanish, "ñ" => "ni"
+    # @param [#to_sym] name The name of the approximations to add.
+    # @param Hash hash The approximations to add.
+    def add_approximations(name, hash)
+      @approximations ||= {}
+      @approximations[name.to_sym] = hash.inject({}) do |memo, object|
+        key = object[0].unpack("U").shift
+        value = object[1].unpack("C*")
+        memo[key] = value.length == 1 ? value[0] : value
+        memo
+      end
+    end
+
+    add_approximations :spanish, "ñ" => "ni"
+    add_approximations :german, "ä" => "ae", "ö" => "oe", "ü" => "ue"
+    add_approximations :latin, {
+      "À" => "A", "Á" => "A", "Â" => "A", "Ã" => "A", "Ä" => "A", "Å" => "A",
+      "Æ" => "AE", "Ç" => "C", "È" => "E", "É" => "E", "Ê" => "E", "Ë" => "E",
+      "Ì" => "I", "Í" => "I", "Î" => "I", "Ï" => "I", "Ð" => "D", "Ñ" => "N",
+      "Ò" => "O", "Ó" => "O", "Ô" => "O", "Õ" => "O", "Ö" => "O", "Ø" => "O",
+      "Ù" => "U", "Ú" => "U", "Û" => "U", "Ü" => "U", "Ý" => "Y", "Þ" => "Th",
+      "ß" => "ss", "à" => "a" , "á" => "a", "â" => "a", "ã" => "a", "ä" => "a",
+      "å" => "a", "æ" => "ae", "ç" => "c" , "è" => "e", "é" => "e", "ê" => "e",
+      "ë" => "e", "ì" => "i", "í" => "i", "î" => "i", "ï" => "i", "ð" => "d",
+      "ñ" => "n", "ò" => "o", "ó" => "o", "ô" => "o", "õ" => "o", "ö" => "o",
+      "ø" => "o", "ù" => "u", "ú" => "u", "û" => "u", "ü" => "u", "ý" => "y",
+      "þ" => "th", "ÿ" => "y", "Ā" => "A", "ā" => "a", "Ă" => "A", "ă" => "a",
+      "Ą" => "A", "ą" => "a", "Ć" => "C", "ć" => "c", "Ĉ" => "C", "ĉ" => "c",
+      "Ċ" => "C", "ċ" => "c", "Č" => "C", "č" => "c", "Ď" => "D", "ď" => "d",
+      "Đ" => "D", "đ" => "d", "Ē" => "E", "ē" => "e", "Ĕ" => "E", "ĕ" => "e",
+      "Ė" => "E", "ė" => "e", "Ę" => "E", "ę" => "e", "Ě" => "E", "ě" => "e",
+      "Ĝ" => "G", "ĝ" => "g", "Ğ" => "G", "ğ" => "g", "Ġ" => "G", "ġ" => "g",
+      "Ģ" => "G", "ģ" => "g", "Ĥ" => "H", "ĥ" => "h", "Ħ" => "H", "ħ" => "h",
+      "Ĩ" => "I", "ĩ" => "i", "Ī" => "I", "ī" => "i", "Ĭ" => "I", "ĭ" => "i",
+      "Į" => "I", "į" => "i", "İ" => "I", "ı" => "i", "IJ" => "IJ", "ij" => "ij",
+      "Ĵ" => "J", "ĵ" => "j", "Ķ" => "K", "ķ" => "k", "ĸ" => "k", "Ĺ" => "L",
+      "ĺ" => "l", "Ļ" => "L", "ļ" => "l", "Ľ" => "L", "ľ" => "l", "Ŀ" => "L",
+      "ŀ" => "l", "Ł" => "L", "ł" => "l", "Ń" => "N", "ń" => "n", "Ņ" => "N",
+      "ņ" => "n", "Ň" => "N", "ň" => "n", "ʼn" => "n", "Ŋ" => "NG", "ŋ" => "ng",
+      "Ō" => "O", "ō" => "o", "Ŏ" => "O", "ŏ" => "o", "Ő" => "O", "ő" => "o",
+      "Œ" => "OE", "œ" => "oe", "Ŕ" => "R", "ŕ" => "r", "Ŗ" => "R", "ŗ" => "r",
+      "Ř" => "R", "ř" => "r", "Ś" => "S", "ś" => "s", "Ŝ" => "S", "ŝ" => "s",
+      "Ş" => "S", "ş" => "s", "Š" => "S", "š" => "s", "Ţ" => "T", "ţ" => "t",
+      "Ť" => "T", "ť" => "t", "Ŧ" => "T", "ŧ" => "t", "Ũ" => "U", "ũ" => "u",
+      "Ū" => "U", "ū" => "u", "Ŭ" => "U", "ŭ" => "u", "Ů" => "U", "ů" => "u",
+      "Ű" => "U", "ű" => "u", "Ų" => "U", "ų" => "u", "Ŵ" => "W", "ŵ" => "w",
+      "Ŷ" => "Y", "ŷ" => "y", "Ÿ" => "Y", "Ź" => "Z", "ź" => "z", "Ż" => "Z",
+      "ż" => "z", "Ž" => "Z", "ž" => "z", "×" => "x", "÷" => "/"
+    }
+  end
+end

data/lib/babosa/slug_string.rb

@@ -0,0 +1,223 @@
+# encoding: utf-8
+
+module Babosa
+
+  # This class provides some string-manipulation methods specific to slugs.
+  #
+  # Note that this class includes many "bang methods" such as {#clean!} and
+  # {#normalize!} that perform actions on the string in-place. Each of these
+  # methods has a corresponding "bangless" method (i.e., +SlugString#clean!+
+  # and +SlugString#clean+) which does not appear in the documentation because
+  # it is generated dynamically.
+  #
+  # All of the bang methods return an instance of String, while the bangless
+  # versions return an instance of Babosa::SlugString, so that calls to methods
+  # specific to this class can be chained:
+  #
+  #   string = SlugString.new("hello world")
+  #   string.with_dashes! # => "hello-world"
+  #   string.with_dashes  # => <Babosa::SlugString:0x000001013e1590 @wrapped_string="hello-world">
+  #
+  # @see http://www.utf8-chartable.de/unicode-utf8-table.pl?utf8=dec Unicode character table
+  class SlugString
+
+    attr_reader :wrapped_string
+    alias to_s wrapped_string
+
+    @@utf8_proxy = if Babosa.jruby15?
+      UTF8::JavaProxy
+    elsif defined? Unicode
+      UTF8::UnicodeProxy
+    elsif defined? ActiveSupport
+      UTF8::ActiveSupportProxy
+    else
+      UTF8::DumbProxy
+    end
+
+    # Return the proxy used for UTF-8 support.
+    # @see Babosa::UTF8::UTF8Proxy
+    def self.utf8_proxy
+      @@utf8_proxy
+    end
+
+    # Set a proxy object used for UTF-8 support.
+    # @see Babosa::UTF8::UTF8Proxy
+    def self.utf8_proxy=(obj)
+      @@utf8_proxy = obj
+    end
+
+    def method_missing(symbol, *args, &block)
+      @wrapped_string.__send__(symbol, *args, &block)
+    end
+
+    # @param string [String] The string to use as the basis of the SlugString.
+    def initialize(string)
+      @wrapped_string = string.to_s
+      tidy_bytes!
+      normalize_utf8!
+    end
+
+    # Approximate an ASCII string. This works only for Western strings using
+    # characters that are Roman-alphabet characters + diacritics. Non-letter
+    # characters are left unmodified.
+    #
+    #   string = SlugString.new "Łódź, Poland"
+    #   string.approximate_ascii                 # => "Lodz, Poland"
+    #   string = SlugString.new "日本"
+    #   string.approximate_ascii                 # => "日本"
+    #
+    # You can pass any key(s) from +Characters.approximations+ as arguments. This allows
+    # for contextual approximations. By default; +:spanish+ and +:german+ are
+    # provided:
+    #
+    #   string = SlugString.new "Jürgen Müller"
+    #   string.approximate_ascii                 # => "Jurgen Muller"
+    #   string.approximate_ascii :german         # => "Juergen Mueller"
+    #   string = SlugString.new "¡Feliz año!"
+    #   string.approximate_ascii                 # => "¡Feliz ano!"
+    #   string.approximate_ascii :spanish        # => "¡Feliz anio!"
+    #
+    # You can modify the built-in approximations, or add your own:
+    #
+    #   # Make Spanish use "nh" rather than "nn"
+    #   Babosa::Characters.add_approximations(:spanish, "ñ" => "nh")
+    #
+    # Notice that this method does not simply convert to ASCII; if you want
+    # to remove non-ASCII characters such as "¡" and "¿", use {#to_ascii!}:
+    #
+    #   string.approximate_ascii!(:spanish)       # => "¡Feliz anio!"
+    #   string.to_ascii!                          # => "Feliz anio!"
+    # @param *args <Symbol>
+    # @return String
+    def approximate_ascii!(overrides = {})
+      overrides = Characters.approximations[overrides] if overrides.kind_of? Symbol
+      @wrapped_string = unpack("U*").map { |char| approx_char(char, overrides) }.flatten.pack("U*")
+    end
+
+    # Converts dashes to spaces, removes leading and trailing spaces, and
+    # replaces multiple whitespace characters with a single space.
+    # @return String
+    def clean!
+      @wrapped_string = @wrapped_string.gsub("-", " ").squeeze(" ").strip
+    end
+
+    # Remove any non-word characters.
+    # @return String
+    def word_chars!
+      @wrapped_string = (unpack("U*") - Characters.strippable).pack("U*")
+    end
+
+    # Normalize the string for use as a slug. Note that in this context,
+    # +normalize+ means, strip, remove non-letters/numbers, downcasing,
+    # truncating to 255 bytes and converting whitespace to dashes.
+    # @param Boolean ascii If true, approximate ASCII and then remove any non-ASCII characters.
+    # @return String
+    def normalize!(ascii = false)
+      if ascii
+        approximate_ascii!
+        to_ascii!
+      end
+      clean!
+      word_chars!
+      clean!
+      downcase!
+      truncate_bytes!(255)
+      with_dashes!
+    end
+
+    # Delete any non-ascii characters.
+    # @return String
+    def to_ascii!
+      @wrapped_string = @wrapped_string.gsub(/[^\x00-\x7f]/u, '')
+    end
+
+    # Truncate the string to +max+ characters.
+    # @example
+    #   "üéøá".to_slug.truncate(3) #=> "üéø"
+    # @return String
+    def truncate!(max)
+      @wrapped_string = unpack("U*")[0...max].pack("U*")
+    end
+
+    # Truncate the string to +max+ bytes. This can be useful for ensuring that
+    # a UTF-8 string will always fit into a database column with a certain max
+    # byte length. The resulting string may be less than +max+ if the string must
+    # be truncated at a multibyte character boundary.
+    # @example
+    #   "üéøá".to_slug.truncate_bytes(3) #=> "ü"
+    # @return String
+    def truncate_bytes!(max)
+      return @wrapped_string if @wrapped_string.bytesize <= max
+      curr = 0
+      new = []
+      unpack("U*").each do |char|
+        break if curr > max
+        char = [char].pack("U")
+        curr += char.bytesize
+        if curr <= max
+          new << char
+        end
+      end
+      @wrapped_string = new.join
+    end
+
+    # Replaces whitespace with dashes ("-").
+    # @return String
+    def with_dashes!
+      @wrapped_string = @wrapped_string.gsub(/\s/u, "-")
+    end
+
+    # Perform UTF-8 sensitive upcasing.
+    # @return String
+    def upcase!
+      @wrapped_string = @@utf8_proxy.upcase(@wrapped_string)
+    end
+
+    # Perform UTF-8 sensitive downcasing.
+    # @return String
+    def downcase!
+      @wrapped_string = @@utf8_proxy.downcase(@wrapped_string)
+    end
+
+    # Perform Unicode composition on the wrapped string.
+    # @return String
+    def normalize_utf8!
+      @wrapped_string = @@utf8_proxy.normalize_utf8(@wrapped_string)
+    end
+
+    # Attempt to convert characters encoded using CP1252 and IS0-8859-1 to
+    # UTF-8.
+    # @return String
+    def tidy_bytes!
+      @wrapped_string = @@utf8_proxy.tidy_bytes(@wrapped_string)
+    end
+
+    %w[approximate_ascii clean downcase word_chars normalize normalize_utf8
+      tidy_bytes to_ascii truncate truncate_bytes upcase with_dashes].each do |method|
+      class_eval(<<-EOM)
+        def #{method}(*args)
+          send_to_new_instance(:#{method}!, *args)
+        end
+      EOM
+    end
+
+    def to_slug
+      self
+    end
+
+    private
+
+    # Look up the character's approximation in the configured maps.
+    def approx_char(char, overrides = {})
+      overrides[char] or Characters.approximations[:latin][char] or char
+    end
+
+    # Used as the basis of the bangless methods.
+    def send_to_new_instance(*args)
+      string = SlugString.new self
+      string.send(*args)
+      string
+    end
+
+  end
+end

data/lib/babosa/utf8/active_support_proxy.rb

@@ -0,0 +1,20 @@
+module Babosa
+  module UTF8
+    # A UTF-8 proxy using Active Support's multibyte support.
+    module ActiveSupportProxy
+      extend UTF8Proxy
+      extend self
+      def downcase(string)
+        ActiveSupport::Multibyte::Chars.new(string).downcase.to_s
+      end
+
+      def upcase(string)
+        ActiveSupport::Multibyte::Chars.new(string).upcase.to_s
+      end
+
+      def normalize_utf8(string)
+        ActiveSupport::Multibyte::Chars.new(string).normalize(:c).to_s
+      end
+    end
+  end
+end

data/lib/babosa/utf8/dumb_proxy.rb

@@ -0,0 +1,41 @@
+require File.expand_path("../mappings", __FILE__)
+module Babosa
+  module UTF8
+
+    # This module provides fallback UTF-8 support when nothing else is
+    # available. It does case folding for Roman alphabet-based characters
+    # commonly used by Western European languages and little else, making it
+    # useless for Russian, Bulgarian, Greek, etc. If at all possible, Unicode
+    # or ActiveSupport should be used instead because they support the full
+    # UTF-8 character range.
+    module DumbProxy
+      extend UTF8Proxy
+      extend self
+
+      def downcase(string)
+        string.unpack("U*").map {|char| Mappings::DOWNCASE[char] or char}.flatten.pack("U*")
+      end
+
+      def upcase(string)
+        string.unpack("U*").map {|char| Mappings::UPCASE[char] or char}.flatten.pack("U*")
+      end
+
+      # This does a very naive Unicode normalization, which should work for
+      # this library's purposes (i.e., Roman-based codepoints, up to U+017E).
+      # Do not use reuse this as a general solution!  Use a real library like
+      # Unicode or ActiveSupport instead.
+      def normalize_utf8(string)
+        codepoints = string.unpack("U*")
+        new = []
+        until codepoints.empty? do
+          if Mappings::COMPOSITION[codepoints[0..1]]
+            new << Mappings::COMPOSITION[codepoints.slice!(0,2)]
+          else
+            new << codepoints.shift
+          end
+        end
+        new.compact.flatten.pack("U*")
+      end
+    end
+  end
+end

data/lib/babosa/utf8/java_proxy.rb

@@ -0,0 +1,24 @@
+include Java
+
+module Babosa
+  module UTF8
+    # A UTF-8 proxy module using Java's built-in Unicode support. Requires JRuby 1.5+.
+    module JavaProxy
+      extend UTF8Proxy
+      extend self
+      import java.text.Normalizer
+
+      def downcase(string)
+        string.to_java.to_lower_case.to_s
+      end
+
+      def upcase(string)
+        string.to_java.to_upper_case.to_s
+      end
+
+      def normalize_utf8(string)
+        Normalizer.normalize(string, Normalizer::Form::NFC).to_s
+      end
+    end
+  end
+end

data/lib/babosa/utf8/mappings.rb

@@ -0,0 +1,193 @@
+module Babosa
+  module UTF8
+
+    # A small subset of the mappings provided by Unicode.org, limited to Latin
+    # characters. This is used for Babosa's default "dumb" UTF-8 support.
+    module Mappings
+      DOWNCASE = Hash[65, 97, 66, 98, 67, 99, 68, 100, 69, 101, 70, 102,
+        71, 103, 72, 104, 73, 105, 74, 106, 75, 107, 76, 108, 77, 109, 78, 110,
+        79, 111, 80, 112, 81, 113, 82, 114, 83, 115, 84, 116, 85, 117, 86, 118,
+        87, 119, 88, 120, 89, 121, 90, 122, 181, 956, 192, 224, 193, 225, 194,
+        226, 195, 227, 196, 228, 197, 229, 198, 230, 199, 231, 200, 232, 201,
+        233, 202, 234, 203, 235, 204, 236, 205, 237, 206, 238, 207, 239, 208,
+        240, 209, 241, 210, 242, 211, 243, 212, 244, 213, 245, 214, 246, 216,
+        248, 217, 249, 218, 250, 219, 251, 220, 252, 221, 253, 222, 254, 223,
+        [115, 115], 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267,
+        268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281,
+        282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295,
+        296, 297, 298, 299, 300, 301, 302, 303, 304, [105, 775], 306, 307, 308,
+        309, 310, 311, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323,
+        324, 325, 326, 327, 328, 329, [700, 110], 330, 331, 332, 333, 334, 335,
+        336, 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349,
+        350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363,
+        364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 255,
+        377, 378, 379, 380, 381, 382]
+
+      UPCASE = DOWNCASE.invert
+
+      COMPOSITION = {
+        [65,768]  => 192,
+        [65,769]  => 193,
+        [65,770]  => 194,
+        [65,771]  => 195,
+        [65,776]  => 196,
+        [65,778]  => 197,
+        [67,807]  => 199,
+        [69,768]  => 200,
+        [69,769]  => 201,
+        [69,770]  => 202,
+        [69,776]  => 203,
+        [73,768]  => 204,
+        [73,769]  => 205,
+        [73,770]  => 206,
+        [73,776]  => 207,
+        [78,771]  => 209,
+        [79,768]  => 210,
+        [79,769]  => 211,
+        [79,770]  => 212,
+        [79,771]  => 213,
+        [79,776]  => 214,
+        [85,768]  => 217,
+        [85,769]  => 218,
+        [85,770]  => 219,
+        [85,776]  => 220,
+        [89,769]  => 221,
+        [97,768]  => 224,
+        [97,769]  => 225,
+        [97,770]  => 226,
+        [97,771]  => 227,
+        [97,776]  => 228,
+        [97,778]  => 229,
+        [99,807]  => 231,
+        [101,768] => 232,
+        [101,769] => 233,
+        [101,770] => 234,
+        [101,776] => 235,
+        [105,768] => 236,
+        [105,769] => 237,
+        [105,770] => 238,
+        [105,776] => 239,
+        [110,771] => 241,
+        [111,768] => 242,
+        [111,769] => 243,
+        [111,770] => 244,
+        [111,771] => 245,
+        [111,776] => 246,
+        [117,768] => 249,
+        [117,769] => 250,
+        [117,770] => 251,
+        [117,776] => 252,
+        [121,769] => 253,
+        [121,776] => 255,
+        [65,772]  => 256,
+        [97,772]  => 257,
+        [65,774]  => 258,
+        [97,774]  => 259,
+        [65,808]  => 260,
+        [97,808]  => 261,
+        [67,769]  => 262,
+        [99,769]  => 263,
+        [67,770]  => 264,
+        [99,770]  => 265,
+        [67,775]  => 266,
+        [99,775]  => 267,
+        [67,780]  => 268,
+        [99,780]  => 269,
+        [68,780]  => 270,
+        [100,780] => 271,
+        [69,772]  => 274,
+        [101,772] => 275,
+        [69,774]  => 276,
+        [101,774] => 277,
+        [69,775]  => 278,
+        [101,775] => 279,
+        [69,808]  => 280,
+        [101,808] => 281,
+        [69,780]  => 282,
+        [101,780] => 283,
+        [71,770]  => 284,
+        [103,770] => 285,
+        [71,774]  => 286,
+        [103,774] => 287,
+        [71,775]  => 288,
+        [103,775] => 289,
+        [71,807]  => 290,
+        [103,807] => 291,
+        [72,770]  => 292,
+        [104,770] => 293,
+        [73,771]  => 296,
+        [105,771] => 297,
+        [73,772]  => 298,
+        [105,772] => 299,
+        [73,774]  => 300,
+        [105,774] => 301,
+        [73,808]  => 302,
+        [105,808] => 303,
+        [73,775]  => 304,
+        [74,770]  => 308,
+        [106,770] => 309,
+        [75,807]  => 310,
+        [107,807] => 311,
+        [76,769]  => 313,
+        [108,769] => 314,
+        [76,807]  => 315,
+        [108,807] => 316,
+        [76,780]  => 317,
+        [108,780] => 318,
+        [78,769]  => 323,
+        [110,769] => 324,
+        [78,807]  => 325,
+        [110,807] => 326,
+        [78,780]  => 327,
+        [110,780] => 328,
+        [79,772]  => 332,
+        [111,772] => 333,
+        [79,774]  => 334,
+        [111,774] => 335,
+        [79,779]  => 336,
+        [111,779] => 337,
+        [82,769]  => 340,
+        [114,769] => 341,
+        [82,807]  => 342,
+        [114,807] => 343,
+        [82,780]  => 344,
+        [114,780] => 345,
+        [83,769]  => 346,
+        [115,769] => 347,
+        [83,770]  => 348,
+        [115,770] => 349,
+        [83,807]  => 350,
+        [115,807] => 351,
+        [83,780]  => 352,
+        [115,780] => 353,
+        [84,807]  => 354,
+        [116,807] => 355,
+        [84,780]  => 356,
+        [116,780] => 357,
+        [85,771]  => 360,
+        [117,771] => 361,
+        [85,772]  => 362,
+        [117,772] => 363,
+        [85,774]  => 364,
+        [117,774] => 365,
+        [85,778]  => 366,
+        [117,778] => 367,
+        [85,779]  => 368,
+        [117,779] => 369,
+        [85,808]  => 370,
+        [117,808] => 371,
+        [87,770]  => 372,
+        [119,770] => 373,
+        [89,770]  => 374,
+        [121,770] => 375,
+        [89,776]  => 376,
+        [90,769]  => 377,
+        [122,769] => 378,
+        [90,775]  => 379,
+        [122,775] => 380,
+        [90,780]  => 381,
+        [122,780] => 382
+      }
+    end
+  end
+end

data/lib/babosa/utf8/proxy.rb

@@ -0,0 +1,118 @@
+module Babosa
+  module UTF8
+
+    autoload :JavaProxy,          "babosa/utf8/java_proxy"
+    autoload :UnicodeProxy,       "babosa/utf8/unicode_proxy"
+    autoload :ActiveSupportProxy, "babosa/utf8/active_support_proxy"
+    autoload :DumbProxy,          "babosa/utf8/dumb_proxy"
+
+    # A UTF-8 proxy for Babosa can be any object which responds to the methods in this module.
+    # The following proxies are provided by Babosa: {ActiveSupportProxy}, {DumbProxy}, {JavaProxy}, and {UnicodeProxy}.
+    module UTF8Proxy
+      CP1252  = {
+        128 => [226, 130, 172],
+        129 => nil,
+        130 => [226, 128, 154],
+        131 => [198, 146],
+        132 => [226, 128, 158],
+        133 => [226, 128, 166],
+        134 => [226, 128, 160],
+        135 => [226, 128, 161],
+        136 => [203, 134],
+        137 => [226, 128, 176],
+        138 => [197, 160],
+        139 => [226, 128, 185],
+        140 => [197, 146],
+        141 => nil,
+        142 => [197, 189],
+        143 => nil,
+        144 => nil,
+        145 => [226, 128, 152],
+        146 => [226, 128, 153],
+        147 => [226, 128, 156],
+        148 => [226, 128, 157],
+        149 => [226, 128, 162],
+        150 => [226, 128, 147],
+        151 => [226, 128, 148],
+        152 => [203, 156],
+        153 => [226, 132, 162],
+        154 => [197, 161],
+        155 => [226, 128, 186],
+        156 => [197, 147],
+        157 => nil,
+        158 => [197, 190],
+        159 => [197, 184]
+      }
+
+      # This is a stub for a method that should return a Unicode-aware
+      # downcased version of the given string.
+      def downcase(string)
+        raise NotImplementedError
+      end
+
+      # This is a stub for a method that should return a Unicode-aware
+      # upcased version of the given string.
+      def upcase(string)
+        raise NotImplementedError
+      end
+
+      # This is a stub for a method that should return the Unicode NFC
+      # normalization of the given string.
+      def normalize_utf8(string)
+        raise NotImplementedError
+      end
+
+      # Attempt to replace invalid UTF-8 bytes with valid ones. This method
+      # naively assumes if you have invalid UTF8 bytes, they are either Windows
+      # CP-1252 or ISO8859-1. In practice this isn't a bad assumption, but may not
+      # always work.
+      def tidy_bytes(string)
+        bytes = string.unpack("C*")
+        conts_expected = 0
+        last_lead = 0
+
+        bytes.each_index do |i|
+          byte          = bytes[i]
+          is_ascii      = byte < 128
+          is_cont       = byte > 127 && byte < 192
+          is_lead       = byte > 191 && byte < 245
+          is_unused     = byte > 240
+          is_restricted = byte > 244
+
+          # Impossible or highly unlikely byte? Clean it.
+          if is_unused || is_restricted
+            bytes[i] = tidy_byte(byte)
+          elsif is_cont
+            # Not expecting contination byte? Clean up. Otherwise, now expect one less.
+            conts_expected == 0 ? bytes[i] = tidy_byte(byte) : conts_expected -= 1
+          else
+            if conts_expected > 0
+              # Expected continuation, but got ASCII or leading? Clean backwards up to
+              # the leading byte.
+              (1..(i - last_lead)).each {|j| bytes[i - j] = tidy_byte(bytes[i - j])}
+              conts_expected = 0
+            end
+            if is_lead
+              # Final byte is leading? Clean it.
+              if i == bytes.length - 1
+                bytes[i] = tidy_byte(bytes.last)
+              else
+                # Valid leading byte? Expect continuations determined by position of
+                # first zero bit, with max of 3.
+                conts_expected = byte < 224 ? 1 : byte < 240 ? 2 : 3
+                last_lead = i
+              end
+            end
+          end
+        end
+        bytes.empty? ? "" : bytes.flatten.compact.pack("C*").unpack("U*").pack("U*")
+      end
+
+      private
+
+      def tidy_byte(byte)
+        byte < 160 ? CP1252[byte] : byte < 192 ? [194, byte] : [195, byte - 64]
+      end
+    end
+  end
+end

data/lib/babosa/utf8/unicode_proxy.rb

@@ -0,0 +1,21 @@
+module Babosa
+  module UTF8
+    # A UTF-8 proxy using the Unicode gem.
+    # @see http://github.com/blackwinter/unicode
+    module UnicodeProxy
+      extend UTF8Proxy
+      extend self
+      def downcase(string)
+        Unicode.downcase(string)
+      end
+
+      def upcase(string)
+        Unicode.upcase(string)
+      end
+
+      def normalize_utf8(string)
+        Unicode.normalize_C(string)
+      end
+    end
+  end
+end

data/lib/babosa/version.rb

@@ -0,0 +1,5 @@
+module Babosa
+  module Version
+    STRING = "0.1.0"
+  end
+end

data/test/babosa_test.rb

@@ -0,0 +1,160 @@
+# encoding: utf-8
+$KCODE = 'UTF8' if RUBY_VERSION < '1.9'
+$LOAD_PATH << File.expand_path("../../lib", __FILE__)
+$LOAD_PATH.uniq!
+
+require "rubygems"
+require "bundler"
+Bundler.setup
+require "test/unit"
+require "babosa"
+
+Module.send :include, Module.new {
+  def test(name, &block)
+    define_method("test_#{name.gsub(/[^a-z0-9_]/i, "_")}".to_sym, &block)
+  end
+}
+
+module UTF8ProxyTest
+  test "should downcase strings" do
+    assert_equal "åéîøü", proxy.downcase("ÅÉÎØÜ")
+  end
+
+  test "should upcase strings" do
+    assert_equal "ÅÉÎØÜ", proxy.upcase("åéîøü")
+  end
+
+  test "should compose UTF-8" do
+    # ÅÉÎØÜ
+    uncomposed_bytes = [65, 204, 138, 69, 204, 129, 73, 204, 130, 195, 152, 85, 204, 136]
+    composed_bytes   = [195, 133, 195, 137, 195, 142, 195, 152, 195, 156]
+    uncomposed_string = uncomposed_bytes.pack("C*").unpack("U*").pack("U*")
+    composed_string   = composed_bytes.pack("C*").unpack("U*").pack("U*")
+    assert_equal composed_bytes, proxy.normalize_utf8(uncomposed_string).unpack("C*")
+  end
+end
+
+if Babosa.jruby15?
+  class JavaProxyTest < Test::Unit::TestCase
+    include UTF8ProxyTest
+    def proxy
+      Babosa::UTF8::JavaProxy
+    end
+  end
+end
+
+class DumbProxyTest < Test::Unit::TestCase
+  include UTF8ProxyTest
+  def proxy
+    Babosa::UTF8::DumbProxy
+  end
+end
+
+class  BabosaTest < Test::Unit::TestCase
+
+  test "word_chars! should leave only letters and spaces" do
+    string = "a*$%^$@!@b$%^&*()*!c"
+    assert_match /[a-z ]*/i, string.to_slug.word_chars!
+  end
+
+  test "approximate_ascii should transliterate to ascii" do
+    slug = (0xC0..0x17E).to_a.each do |codepoint|
+      ss = [codepoint].pack("U*").to_slug
+      approx = ss.approximate_ascii
+      assert_match /[\x0-\x7f]/, approx.to_s
+    end
+  end
+
+  test "should lowercase strings" do
+    assert_equal "feliz año", "FELIZ AÑO".to_slug.downcase!
+  end
+
+  test "should uppercase strings" do
+    assert_equal "FELIZ AÑO", "feliz año".to_slug.upcase!
+  end
+
+  test "should replace whitespace with dashes" do
+    assert_equal "a-b", "a b".to_slug.clean.normalize!
+  end
+
+  test "should replace multiple spaces with 1 dash" do
+    assert_equal "a-b", "a    b".to_slug.clean.normalize!
+  end
+
+  test "should replace multiple dashes with 1 dash" do
+    assert_equal "male-female", "male - female".to_slug.normalize!
+  end
+
+  test "should strip trailing space" do
+    assert_equal "ab", "ab ".to_slug.normalize!
+  end
+
+  test "should strip leading space" do
+    assert_equal "ab", " ab".to_slug.normalize!
+  end
+
+  test "should strip trailing slashes" do
+    assert_equal "ab", "ab-".to_slug.normalize!
+  end
+
+  test "should strip leading slashes" do
+    assert_equal "ab", "-ab".to_slug.normalize!
+  end
+
+  test "should not modify valid name strings" do
+    assert_equal "a-b-c-d", "a-b-c-d".to_slug.normalize!
+  end
+
+  test "should do special approximations for German" do
+    assert_equal "Juergen", "Jürgen".to_slug.approximate_ascii!(:german)
+  end
+
+  test "should do special approximations for Spanish" do
+    assert_equal "anio", "año".to_slug.approximate_ascii!(:spanish)
+  end
+
+  test "should work with non roman chars" do
+    assert_equal "検-索", "検 索".to_slug.normalize!
+  end
+
+  test "should work with invalid UTF-8 strings" do
+    %w[approximate_ascii clean downcase word_chars normalize to_ascii upcase with_dashes].each do |method|
+      string = "\x93abc".to_slug
+      assert_nothing_raised do
+        method == "truncate" ? string.send(method, 32) : string.send(method)
+      end
+    end
+  end
+
+  test "should truncate string by byte length" do
+    assert_equal "ü",  "üa".to_slug.truncate_bytes!(2)
+    assert_equal "",   "üa".to_slug.truncate_bytes!(1)
+    assert_equal "üa", "üa".to_slug.truncate_bytes!(100)
+    assert_equal "ü",  "üéøá".to_slug.truncate_bytes!(3)
+  end
+
+  test "should truncate string by char length" do
+    assert_equal "üa", "üa".to_slug.truncate!(2)
+    assert_equal "ü",  "üa".to_slug.truncate!(1)
+    assert_equal "üa", "üa".to_slug.truncate!(100)
+  end
+
+  test "should transliterate uncomposed utf8" do
+    string = [117, 776].pack("U*") # "ü" as ASCII "u" plus COMBINING DIAERESIS
+    assert_equal "u", string.to_slug.approximate_ascii!
+  end
+
+  test "with_dashes should not change byte size when replacing spaces" do
+    assert_equal "".bytesize, "".to_slug.with_dashes.bytesize
+    assert_equal " ".bytesize, " ".to_slug.with_dashes.bytesize
+    assert_equal "-abc-".bytesize, "-abc-".to_slug.with_dashes.bytesize
+    assert_equal " abc ".bytesize, " abc ".to_slug.with_dashes.bytesize
+    assert_equal " a  bc ".bytesize, " a  bc ".to_slug.with_dashes.bytesize
+  end
+
+  test "normalize! with ascii should approximate and strip non ascii" do
+    ss = "カタカナ: katakana is über cool".to_slug
+    assert_equal "katakana-is-uber-cool", ss.normalize!(true)
+  end
+
+end

metadata

@@ -0,0 +1,78 @@
+--- !ruby/object:Gem::Specification 
+name: babosa
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Norman Clarke
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-07-12 00:00:00 -03:00
+default_executable: 
+dependencies: []
+
+description: "    A library for creating slugs. Babosa an extraction and improvement of the\n    string code from FriendlyId, intended to help developers create similar\n    libraries or plugins.\n"
+email: norman@njclarke.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/babosa/characters.rb
+- lib/babosa/slug_string.rb
+- lib/babosa/utf8/active_support_proxy.rb
+- lib/babosa/utf8/dumb_proxy.rb
+- lib/babosa/utf8/java_proxy.rb
+- lib/babosa/utf8/mappings.rb
+- lib/babosa/utf8/proxy.rb
+- lib/babosa/utf8/unicode_proxy.rb
+- lib/babosa/version.rb
+- lib/babosa.rb
+- README.md
+- MIT-LICENSE
+- Rakefile
+- init.rb
+- test/babosa_test.rb
+has_rdoc: false
+homepage: http://norman.github.com/babosa
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: "[none]"
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: A library for creating slugs.
+test_files: 
+- test/babosa_test.rb