stringex 2.0.11 → 2.8.6

data/lib/stringex/localization/conversion_expressions.rb

@@ -3,43 +3,44 @@
 module Stringex
   module Localization
     module ConversionExpressions
-      ABBREVIATION = /(\s|^)([[:alpha:]](\.[[:alpha:]])+(\.?)[[:alpha:]]*(\s|$))/
+      ABBREVIATION = /(\s|\(|^)([[:alpha:]](\.[[:alpha:]])+(\.?)[[:alpha:]]*(\s|\)|$))/
 
       ACCENTED_HTML_ENTITY = /&([A-Za-z])(grave|acute|circ|tilde|uml|ring|cedil|slash);/
 
       APOSTROPHE = /(^|[[:alpha:]])'|`([[:alpha:]]|$)/
 
       CHARACTERS =  {
-        :and           => /\s*&\s*/,
-        :at            => /\s*@\s*/,
-        :degrees       => /\s*°\s*/,
-        :divide        => /\s*÷\s*/,
-        :dot           => /(\S|^)\.(\S)/,
-        :ellipsis      => /\s*\.{3,}\s*/,
-        :equals        => /\s*=\s*/,
-        :number        => /\s*#/,
-        :percent       => /\s*%\s*/,
-        :plus          => /\s*\+\s*/,
-        :slash         => /\s*(\\|\/|／)\s*/,
-        :star          => /\s*\*\s*/,
+        and:      /\s*&\s*/,
+        at:       /\s*@\s*/,
+        degrees:  /\s*°\s*/,
+        divide:   /\s*÷\s*/,
+        dot:      /(\S|^)\.(\S)/,
+        ellipsis: /\s*\.{3,}\s*/,
+        equals:   /\s*=\s*/,
+        number:   /\s*#/,
+        percent:  /\s*%\s*/,
+        plus:     /\s*\+\s*/,
+        slash:    /\s*(\\|\/|／)\s*/,
+        star:     /\s*\*\s*/,
       }
 
       # Things that just get converted to spaces
-      CLEANUP_CHARACTERS = /[\.,:;(){}\[\]\?!\^'ʼ"_\|<>]/
+      CLEANUP_CHARACTERS = /[\.,:;(){}\[\]\?!\^'ʼ"`~_\|<>]/
       CLEANUP_HTML_ENTITIES = /&[^;]+;/
 
       CURRENCIES_SUPPORTED_SIMPLE = {
-        :dollars => /\$/,
-        :euros   => /€/,
-        :pounds  => /£/,
-        :yen     => /¥/,
-        :reais   => /R\$/
+        generic: /¤/,
+        dollars: /\$/,
+        euros:   /€/,
+        pounds:  /£/,
+        yen:     /¥/,
+        reais:   /R\$/
       }
       CURRENCIES_SUPPORTED_COMPLEX = {
-        :dollars => :dollars_cents,
-        :euros   => :euros_cents,
-        :pounds  => :pounds_pence,
-        :reais   => :reais_cents
+        dollars: :dollars_cents,
+        euros:   :euros_cents,
+        pounds:  :pounds_pence,
+        reais:   :reais_cents
       }
       CURRENCIES_SUPPORTED = Regexp.new(CURRENCIES_SUPPORTED_SIMPLE.values.join('|'))
       CURRENCIES_SIMPLE = CURRENCIES_SUPPORTED_SIMPLE.inject({}) do |hash, content|
@@ -60,27 +61,27 @@ module Stringex
 
       HTML_ENTITIES = Proc.new(){
         base = {
-          :amp          => %w{#38 amp},
-          :cent         => %w{#162 cent},
-          :copy         => %w{#169 copy},
-          :deg          => %w{#176 deg},
-          :divide       => %w{#247 divide},
-          :double_quote => %w{#34 #822[012] quot ldquo rdquo dbquo},
-          :ellipsis     => %w{#8230 hellip},
-          :en_dash      => %w{#8211 ndash},
-          :em_dash      => %w{#8212 mdash},
-          :frac14       => %w{#188 frac14},
-          :frac12       => %w{#189 frac12},
-          :frac34       => %w{#190 frac34},
-          :gt           => %w{#62 gt},
-          :lt           => %w{#60 lt},
-          :nbsp         => %w{#160 nbsp},
-          :pound        => %w{#163 pound},
-          :reg          => %w{#174 reg},
-          :single_quote => %w{#39 #821[678] apos lsquo rsquo sbquo},
-          :times        => %w{#215 times},
-          :trade        => %w{#8482 trade},
-          :yen          => %w{#165 yen},
+          amp:          %w{#38 amp},
+          cent:         %w{#162 cent},
+          copy:         %w{#169 copy},
+          deg:          %w{#176 deg},
+          divide:       %w{#247 divide},
+          double_quote: %w{#34 #822[012] quot ldquo rdquo dbquo},
+          ellipsis:     %w{#8230 hellip},
+          en_dash:      %w{#8211 ndash},
+          em_dash:      %w{#8212 mdash},
+          frac14:       %w{#188 frac14},
+          frac12:       %w{#189 frac12},
+          frac34:       %w{#190 frac34},
+          gt:           %w{#62 gt},
+          lt:           %w{#60 lt},
+          nbsp:         %w{#160 nbsp},
+          pound:        %w{#163 pound},
+          reg:          %w{#174 reg},
+          single_quote: %w{#39 #821[678] apos lsquo rsquo sbquo},
+          times:        %w{#215 times},
+          trade:        %w{#8482 trade},
+          yen:          %w{#165 yen},
         }
         base.inject({}) do |hash, content|
           key, expression = content
@@ -90,7 +91,7 @@ module Stringex
       }.call
 
       HTML_TAG = Proc.new(){
-        name = /[\w:_-]+/
+        name = /[\w:-]+/
         value = /([A-Za-z0-9]+|('[^']*?'|"[^"]*?"))/
         attr = /(#{name}(\s*=\s*#{value})?)/
         /<[!\/?\[]?(#{name}|--)(\s+(#{attr}(\s+#{attr})*))?\s*([!\/?\]]+|--)?>/
@@ -102,23 +103,25 @@ module Stringex
         /…/ => "...",
       }
 
+      UNREADABLE_CONTROL_CHARACTERS = /[[:cntrl:]]/
+
       # Ordered by denominator then numerator of the value
       VULGAR_FRACTIONS = {
-        :half          => /(&#189;|&frac12;|½)/,
-        :one_third     => /(&#8531;|⅓)/,
-        :two_thirds    => /(&#8532;|⅔)/,
-        :one_fourth    => /(&#188;|&frac14;|¼)/,
-        :three_fourths => /(&#190;|&frac34;|¾)/,
-        :one_fifth     => /(&#8533;|⅕)/,
-        :two_fifths    => /(&#8534;|⅖)/,
-        :three_fifths  => /(&#8535;|⅗)/,
-        :four_fifths   => /(&#8536;|⅘)/,
-        :one_sixth     => /(&#8537;|⅙)/,
-        :five_sixths   => /(&#8538;|⅚)/,
-        :one_eighth    => /(&#8539;|⅛)/,
-        :three_eighths => /(&#8540;|⅜)/,
-        :five_eighths  => /(&#8541;|⅝)/,
-        :seven_eighths => /(&#8542;|⅞)/,
+        half:          /(&#189;|&frac12;|½)/,
+        one_third:     /(&#8531;|⅓)/,
+        two_thirds:    /(&#8532;|⅔)/,
+        one_fourth:    /(&#188;|&frac14;|¼)/,
+        three_fourths: /(&#190;|&frac34;|¾)/,
+        one_fifth:     /(&#8533;|⅕)/,
+        two_fifths:    /(&#8534;|⅖)/,
+        three_fifths:  /(&#8535;|⅗)/,
+        four_fifths:   /(&#8536;|⅘)/,
+        one_sixth:     /(&#8537;|⅙)/,
+        five_sixths:   /(&#8538;|⅚)/,
+        one_eighth:    /(&#8539;|⅛)/,
+        three_eighths: /(&#8540;|⅜)/,
+        five_eighths:  /(&#8541;|⅝)/,
+        seven_eighths: /(&#8542;|⅞)/,
       }
 
       WHITESPACE = /\s+/
@@ -137,6 +140,7 @@ module Stringex
           html_entities
           html_tag
           smart_punctuation
+          unreadable_control_characters
           vulgar_fractions
           whitespace
         }.each do |conversion_type|

data/lib/stringex/localization/converter.rb

@@ -37,7 +37,7 @@ module Stringex
       end
 
       def normalize_currency!
-        string.gsub! /(\d+),(\d+)/, '\1\2'
+        string.gsub!(/(\d+),(\d+)/, '\1\2')
       end
 
       def smart_strip!
@@ -59,7 +59,11 @@ module Stringex
         end
       end
 
-    protected
+      protected
+
+      def unreadable_control_characters
+        string.gsub! expressions.unreadable_control_characters, ''
+      end
 
       def abbreviations
         string.gsub! expressions.abbreviation do |x|
@@ -107,7 +111,7 @@ module Stringex
         end
       end
 
-    private
+      private
 
       def expressions
         ConversionExpressions
@@ -122,4 +126,4 @@ module Stringex
       end
     end
   end
-end
+end

data/lib/stringex/localization/default_conversions.rb

@@ -4,76 +4,77 @@ module Stringex
   module Localization
     module DefaultConversions
       CHARACTERS =  {
-        :and           => "and",
-        :at            => "at",
-        :degrees       => "degrees",
-        :divide        => "divided by",
-        :dot           => '\1 dot \2',
-        :ellipsis      => "dot dot dot",
-        :equals        => "equals",
-        :number        => "number",
-        :percent       => "percent",
-        :plus          => "plus",
-        :slash         => "slash",
-        :star          => "star",
+        and:      "and",
+        at:       "at",
+        degrees:  "degrees",
+        divide:   "divided by",
+        dot:      '\1 dot \2',
+        ellipsis: "dot dot dot",
+        equals:   "equals",
+        number:   "number",
+        percent:  "percent",
+        plus:     "plus",
+        slash:    "slash",
+        star:     "star",
       }
 
       CURRENCIES_SIMPLE = {
-        :dollars       => '\1 dollars',
-        :euros         => '\1 euros',
-        :pounds        => '\1 pounds',
-        :yen           => '\1 yen',
+        generic: '\1 dollars',
+        dollars: '\1 dollars',
+        euros:   '\1 euros',
+        pounds:  '\1 pounds',
+        yen:     '\1 yen',
       }
       CURRENCIES_COMPLEX = {
-        :dollars_cents => '\1 dollars \2 cents',
-        :euros_cents   => '\1 euros \2 cents',
-        :pounds_pence  => '\1 pounds \2 pence',
+        dollars_cents: '\1 dollars \2 cents',
+        euros_cents:   '\1 euros \2 cents',
+        pounds_pence:  '\1 pounds \2 pence',
       }
       CURRENCIES = CURRENCIES_SIMPLE.merge(CURRENCIES_COMPLEX)
 
       HTML_ENTITIES = {
-        :amp          => "and",
-        :cent         => " cents",
-        :copy         => "(c)",
-        :deg          => " degrees ",
-        :divide       => " divided by ",
-        :double_quote => '"',
-        :ellipsis     => "...",
-        :en_dash      => "-",
-        :em_dash      => "--",
-        :frac14       => "one fourth",
-        :frac12       => "half",
-        :frac34       => "three fourths",
-        :gt           => ">",
-        :lt           => "<",
-        :nbsp         => " ",
-        :pound        => " pounds ",
-        :reg          => "(r)",
-        :single_quote => "'",
-        :times        => "x",
-        :trade        => "(tm)",
-        :yen          => " yen "
+        amp:          "and",
+        cent:         " cents",
+        copy:         "(c)",
+        deg:          " degrees ",
+        divide:       " divided by ",
+        double_quote: '"',
+        ellipsis:     "...",
+        en_dash:      "-",
+        em_dash:      "--",
+        frac14:       "one fourth",
+        frac12:       "half",
+        frac34:       "three fourths",
+        gt:           ">",
+        lt:           "<",
+        nbsp:         " ",
+        pound:        " pounds ",
+        reg:          "(r)",
+        single_quote: "'",
+        times:        "x",
+        trade:        "(tm)",
+        yen:          " yen "
       }
 
       TRANSLITERATIONS = {}
 
       # Ordered by denominator then numerator of the value
       VULGAR_FRACTIONS = {
-        :half          => "half",
-        :one_third     => "one third",
-        :two_thirds    => "two thirds",
-        :one_fourth    => "one fourth",
-        :three_fourths => "three fourths",
-        :one_fifth     => "one fifth",
-        :two_fifths    => "two fifths",
-        :three_fifths  => "three fifths",
-        :four_fifths   => "four fifths",
-        :one_sixth     => "one sixth",
-        :five_sixths   => "five sixths",
-        :one_eighth    => "one eighth",
-        :three_eighths => "three eighths",
-        :five_eighths  => "five eighths",
-        :seven_eighths => "seven eighths",
+        half:          "half",
+        one_third:     "one third",
+        two_thirds:    "two thirds",
+        one_fourth:    "one fourth",
+        three_fourths: "three fourths",
+        one_fifth:     "one fifth",
+        two_fifths:    "two fifths",
+        three_fifths:  "three fifths",
+        four_fifths:   "four fifths",
+        one_sixth:     "one sixth",
+        five_sixths:   "five sixths",
+        one_eighth:    "one eighth",
+        three_eighths: "three eighths",
+        five_eighths:  "five eighths",
+        seven_eighths: "seven eighths",
       }
 
       class << self

data/lib/stringex/localization.rb

@@ -45,7 +45,7 @@ module Stringex
         return translation unless translation.nil?
 
         if locale != default_locale
-          translate scope, key, options.merge(:locale => default_locale)
+          translate scope, key, options.merge(locale: default_locale)
         else
           default_conversion(scope, key) || options[:default]
         end
@@ -83,7 +83,7 @@ module Stringex
 
       def convert(string, options = {}, &block)
         converter = Converter.new(string, options)
-        converter.instance_exec &block
+        converter.instance_exec(&block)
         converter.smart_strip!
         converter.string
       end

data/lib/stringex/string_extensions.rb

@@ -3,7 +3,7 @@
 module Stringex
   module StringExtensions
     def self.configure(&block)
-      Stringex::Configuration::StringExtensions.configure &block
+      Stringex::Configuration::StringExtensions.configure(&block)
     end
 
     def self.unconfigure!
@@ -52,7 +52,7 @@ module Stringex
       # It allows localization of conversions so you can use it to convert characters into your own language.
       # Example:
       #
-      #   I18n.backend.store_translations :de, { :stringex => { :characters => { :and => "und" } } }
+      #   I18n.backend.store_translations :de, { stringex: { characters: { and: "und" } } }
       #   I18n.locale = :de
       #   "ich & dich".convert_misc_characters # => "ich und dich"
       #
@@ -93,11 +93,38 @@ module Stringex
         end
       end
 
+      def convert_unreadable_control_characters
+        stringex_convert do
+          translate! :unreadable_control_characters
+        end
+      end
+
       # Returns the string limited in size to the value of limit.
-      def limit(limit = nil)
-        limit.nil? ? self : self[0...limit]
+      def limit(limit = nil, truncate_words = true, whitespace_replacement_token = "-")
+        if limit.nil?
+          self
+        else
+          truncate_words == false ? self.whole_word_limit(limit, whitespace_replacement_token) : self[0...limit]
+        end
       end
 
+      def whole_word_limit(limit, whitespace_replacement_token = "-")
+        whole_words = []
+        words = self.split(whitespace_replacement_token)
+
+        words.each do |word|
+          if word.size > limit
+            break
+          else
+            whole_words << word
+            limit -= (word.size + 1)
+          end
+        end
+
+        whole_words.join(whitespace_replacement_token)
+      end
+
+
       # Performs multiple text manipulations. Essentially a shortcut for typing them all. View source
       # below to see which methods are run.
       def remove_formatting(options = {})
@@ -105,6 +132,7 @@ module Stringex
           convert_smart_punctuation.
           convert_accented_html_entities.
           convert_vulgar_fractions.
+          convert_unreadable_control_characters.
           convert_miscellaneous_html_entities.
           convert_miscellaneous_characters(options).
           to_ascii.
@@ -166,8 +194,8 @@ module Stringex
         whitespace_replacement_token = options[:replace_whitespace_with]
         dummy = remove_formatting(options).
                   replace_whitespace(whitespace_replacement_token).
-                  collapse("-").
-                  limit(options[:limit])
+                  collapse(whitespace_replacement_token).
+                  limit(options[:limit], options[:truncate_words], whitespace_replacement_token)
         dummy.downcase! unless options[:force_downcase] == false
         dummy
       end

data/lib/stringex/unidecoder.rb

@@ -1,6 +1,7 @@
 # encoding: UTF-8
 
 require 'yaml'
+require 'stringex/localization'
 
 module Stringex
   module Unidecoder
@@ -14,20 +15,7 @@ module Stringex
       #
       # You're probably better off just using the added String#to_ascii
       def decode(string)
-        string.gsub(/[^\x00-\x00]/u) do |codepoint|
-          if localized = translate(codepoint)
-            localized
-          else
-            begin
-              unpacked = codepoint.unpack("U")[0]
-              CODEPOINTS[code_group(unpacked)][grouped_point(unpacked)]
-            rescue
-              # Hopefully this won't come up much
-              # TODO: Make this note something to the user that is reportable to me perhaps
-              "?"
-            end
-          end
-        end
+        string.chars.map{|char| decoded(char)}.join
       end
 
       # Returns character for the given Unicode codepoint
@@ -49,8 +37,22 @@ module Stringex
 
     private
 
-      def translate(codepoint)
-        Localization.translate(:transliterations, codepoint)
+      def decoded(character)
+        localized(character) || from_yaml(character)
+      end
+
+      def localized(character)
+        Localization.translate(:transliterations, character)
+      end
+
+      def from_yaml(character)
+        return character unless character.ord > 128
+        unpacked = character.unpack("U")[0]
+        CODEPOINTS[code_group(unpacked)][grouped_point(unpacked)]
+      rescue
+        # Hopefully this won't come up much
+        # TODO: Make this note something to the user that is reportable to me perhaps
+        "?"
       end
 
       # Returns the Unicode codepoint grouping for the given character
@@ -67,12 +69,14 @@ module Stringex
 end
 
 module Stringex
-  module StringExtensions::PublicInstanceMethods
-    # Returns string with its UTF-8 characters transliterated to ASCII ones. Example:
-    #
-    #   "⠋⠗⠁⠝⠉⠑".to_ascii #=> "france"
-    def to_ascii
-      Stringex::Unidecoder.decode(self)
+  module StringExtensions
+    module PublicInstanceMethods
+      # Returns string with its UTF-8 characters transliterated to ASCII ones. Example:
+      #
+      #   "⠋⠗⠁⠝⠉⠑".to_ascii #=> "france"
+      def to_ascii
+        Stringex::Unidecoder.decode(self)
+      end
     end
   end
 end

data/lib/stringex/unidecoder_data/x00.yml

@@ -163,7 +163,7 @@
 - ''
 - C/
 - PS
-- $?
+- $
 - Y=
 - '|'
 - SS

data/lib/stringex/unidecoder_data/x03.yml

@@ -166,7 +166,7 @@
 - T
 - Y
 - Ph
-- Ks
+- X
 - Ps
 - O
 - I
@@ -253,4 +253,4 @@
 - '[?]'
 - '[?]'
 - '[?]'
-- '[?]'
+- '[?]'

data/lib/stringex/unidecoder_data/x05.yml

@@ -137,7 +137,7 @@
 - ew
 - '[?]'
 - .
-- -
+- '-'
 - '[?]'
 - '[?]'
 - '[?]'

data/lib/stringex/unidecoder_data/x07.yml

@@ -4,7 +4,7 @@
 - ','
 - '!'
 - '!'
-- -
+- '-'
 - ','
 - ','
 - ;

data/lib/stringex/unidecoder_data/x09.yml

@@ -16,7 +16,7 @@
 - e
 - e
 - ai
-- oN
+- 'oN'
 - o
 - o
 - au
@@ -72,7 +72,7 @@
 - e
 - e
 - ai
-- oN
+- 'oN'
 - o
 - o
 - au

data/lib/stringex/unidecoder_data/x0a.yml

@@ -144,7 +144,7 @@
 - '[?]'
 - e
 - ai
-- oN
+- 'oN'
 - '[?]'
 - o
 - au
@@ -200,7 +200,7 @@
 - '[?]'
 - e
 - ai
-- oN
+- 'oN'
 - '[?]'
 - o
 - au

data/lib/stringex/unidecoder_data/x0f.yml

@@ -10,7 +10,7 @@
 - ' // '
 - ' * '
 - ''
-- -
+- '-'
 - ' / '
 - ' / '
 - ' // '

data/lib/stringex/unidecoder_data/x12.yml

@@ -149,7 +149,7 @@
 - naa
 - nee
 - ne
-- no
+- 'no'
 - nwa
 - nya
 - nyu

data/lib/stringex/unidecoder_data/x13.yml

@@ -194,7 +194,7 @@
 - nah
 - ne
 - ni
-- no
+- 'no'
 - nu
 - nv
 - qua

data/lib/stringex/unidecoder_data/x14.yml

@@ -44,7 +44,7 @@
 - l
 - en
 - in
-- on
+- 'on'
 - an
 - pe
 - paai
@@ -195,7 +195,7 @@
 - naai
 - ni
 - nii
-- no
+- 'no'
 - noo
 - noo
 - na

data/lib/stringex/unidecoder_data/x15.yml

@@ -185,7 +185,7 @@
 - wa
 - ne
 - ni
-- no
+- 'no'
 - na
 - ke
 - ki