twitter_cldr 1.9.1 → 2.0.0

data/lib/twitter_cldr/tokenizers/numbers/number_tokenizer.rb

@@ -6,54 +6,77 @@
 module TwitterCldr
   module Tokenizers
     class NumberTokenizer < Base
-      VALID_TYPES = [:decimal, :percent, :currency]
+      ABBREVIATED_MIN_POWER = 3
+      ABBREVIATED_MAX_POWER = 14
+
+      VALID_TYPES = [:decimal, :percent, :currency, :short_decimal, :long_decimal]
+      TOKEN_SPLITTER_REGEX = /([^0*#,\.]*)([0#,\.]+)([^0*#,\.]*)$/
+      TOKEN_TYPE_REGEXES = {
+        :pattern => { :regex => /[0?#,\.]*/, :priority => 1 },
+        :plaintext => { :regex => //, :priority => 2 }
+      }
 
       def initialize(options = {})
         super(options)
 
-        @type = options[:type] || :decimal
+        @token_splitter_regexes = {
+          :else => TOKEN_SPLITTER_REGEX
+        }
 
-        @token_splitter_regex = /([^0*#,\.]*)([0#,\.]+)([^0*#,\.]*)$/
-        @token_type_regexes   = [{ :type => :pattern, :regex => /[0?#,\.]*/ }, { :type => :plaintext, :regex => // }]
+        @token_type_regexes = {
+          :else => TOKEN_TYPE_REGEXES
+        }
 
         @base_path   = [:numbers, :formats]
         @symbol_path = [:numbers, :symbols]
 
         @paths = {
-            :default  => [:patterns, :default],
-            :positive => [:patterns, :positive],
-            :negative => [:patterns, :negative]
+          :default       => [:decimal, :patterns],
+          :decimal       => [:decimal, :patterns],
+          :long_decimal  => [:decimal, :patterns, :long],
+          :short_decimal => [:decimal, :patterns, :short],
+          :currency      => [:currency, :patterns],
+          :percent       => [:percent, :patterns]
         }
       end
 
       def tokens(options = {})
-        unless traverse(full_path_for(:positive))
-          key_path = full_path_for(:default)
+        @type = options[:type] || @type || :default
+        @format = options[:format] || @format || :default
 
-          positive, negative = traverse(key_path).split(/;/)
-
-          insert_point = traverse(key_path[0..-2])
-          insert_point[:positive] = positive
+        path = full_path
+        positive, negative = traverse(path).to_s.split(/;/)
+        sign = options[:sign] || :positive
 
-          if negative
-            insert_point[:negative] = "#{symbols[:minus] || '-'}#{negative}"
+        pattern = case sign
+          when :negative
+            if negative
+              "#{symbols[:minus] || '-'}#{negative}"
+            else
+              "#{symbols[:minus] || '-'}#{positive}"
+            end
           else
-            insert_point[:negative] = "#{symbols[:minus] || '-'}#{positive}"
-          end
+            positive
         end
 
-        sign = options[:sign] || :positive
-        tokens_for(full_path_for(sign), nil)
+        tokens_for_pattern(pattern, path, [sign])
       end
 
       def symbols
-        self.traverse(@symbol_path)
+        traverse(@symbol_path)
+      end
+
+      def valid_types
+        VALID_TYPES.select do |type|
+          result = traverse(@base_path + @paths[type])
+          !!result ? result.size > 0 : result
+        end
       end
 
       protected
 
-      def full_path_for(path)
-        @base_path + [@type] + paths[path]
+      def full_path
+        @base_path + paths[@type] + [@format]
       end
 
       def init_resources

data/lib/twitter_cldr/utils.rb

@@ -29,11 +29,26 @@ module TwitterCldr
         elsif first.is_a?(Array) && second.is_a?(Array)
           second.each_with_index { |elem, index| first[index] = deep_merge!(first[index], elem) }
         else
-         return second
+          return second
         end
         first
       end
 
+      def deep_merge_hash(first, second, &block)
+        target = first.dup
+
+        second.keys.each do |key|
+          if second[key].is_a?(Hash) && first[key].is_a?(Hash)
+            target[key] = deep_merge_hash(target[key], second[key], &block)
+            next
+          end
+
+          target[key] = block_given? ? yield(first[key], second[key]) : second[key]
+        end
+
+        target
+      end
+
       def compute_cache_key(*pieces)
         if pieces && pieces.size > 0
           pieces.join("|").hash

data/lib/twitter_cldr/version.rb

@@ -4,5 +4,5 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 
 module TwitterCldr
-  VERSION = "1.9.1"
+  VERSION = "2.0.0"
 end

data/resources/collation/tailoring/af.yml

@@ -2,3 +2,4 @@
 :collator_options: {}
 :tailored_table: 0149; [41, 5, 10]
 :suppressed_contractions: ''
+...

data/resources/collation/tailoring/ar.yml

@@ -1,6 +1,6 @@
 ---
 :collator_options: {}
-:tailored_table: 0629; [6144, 87, 5]
+:tailored_table: ! '0629; [6144, 87, 5]
 
   FE94; [6144, 87, 6]
 
@@ -18,5 +18,6 @@
 
   FC90; [6287, 87, A]
 
-  FC5D; [6287, 87, B]
+  FC5D; [6287, 87, B]'
 :suppressed_contractions: ''
+...

data/resources/collation/tailoring/be.yml

@@ -0,0 +1,7 @@
+---
+:collator_options: {}
+:tailored_table: ! '0435 0308; [5C35, 5, 5]
+
+  0415 0308; [5C35, 5, 86]'
+:suppressed_contractions: ІАГЕЖЗКОЧЫЭагежзкочыэіѴѵӘәӨө
+...

data/resources/collation/tailoring/bg.yml

@@ -0,0 +1,5 @@
+---
+:collator_options: {}
+:tailored_table: ''
+:suppressed_contractions: ІАГЕЖЗКОУЧЫЭагежзкоучыэіѴѵӘәӨө
+...

data/resources/collation/tailoring/bn.yml

@@ -0,0 +1,9 @@
+---
+:collator_options: {}
+:tailored_table: ! '0982; [6925, 5, 5][4, 0, C0]
+
+  0983; [6925, 5, 5][5, 0, C0]
+
+  0981; [6925, 5, 5][6, 0, C0]'
+:suppressed_contractions: ''
+...

data/resources/collation/tailoring/ca.yml

@@ -1,10 +1,11 @@
 ---
 :collator_options: {}
-:tailored_table: 0140; [3D, 87, 5]
+:tailored_table: ! '0140; [3D, 87, 5]
 
   006C 00B7; [3D, 87, 5]
 
   013F; [3D, 87, 86]
 
-  004C 00B7; [3D, 87, 86]
+  004C 00B7; [3D, 87, 86]'
 :suppressed_contractions: ''
+...

data/resources/collation/tailoring/cs.yml

@@ -1,6 +1,6 @@
 ---
 :collator_options: {}
-:tailored_table: 0063 030C; [2C04, 5, 5]
+:tailored_table: ! '0063 030C; [2C04, 5, 5]
 
   0043 030C; [2C04, 5, 86]
 
@@ -22,5 +22,6 @@
 
   007A 030C; [5A04, 5, 5]
 
-  005A 030C; [5A04, 5, 86]
+  005A 030C; [5A04, 5, 86]'
 :suppressed_contractions: ''
+...

data/resources/collation/tailoring/cy.yml

@@ -0,0 +1,51 @@
+---
+:collator_options: {}
+:tailored_table: ! '0063 0068; [2C04, 5, 5]
+
+  0043 0068; [2C04, 5, 46]
+
+  0043 0048; [2C04, 5, 87]
+
+  0064 0064; [2E04, 5, 5]
+
+  0044 0064; [2E04, 5, 46]
+
+  0044 0044; [2E04, 5, 87]
+
+  0066 0066; [3204, 5, 5]
+
+  0046 0066; [3204, 5, 46]
+
+  0046 0046; [3204, 5, 87]
+
+  006E 0067; [3404, 5, 5]
+
+  004E 0067; [3404, 5, 46]
+
+  004E 0047; [3404, 5, 87]
+
+  006C 006C; [3E04, 5, 5]
+
+  004C 006C; [3E04, 5, 46]
+
+  004C 004C; [3E04, 5, 87]
+
+  0070 0068; [4604, 5, 5]
+
+  0050 0068; [4604, 5, 46]
+
+  0050 0048; [4604, 5, 87]
+
+  0072 0068; [4A04, 5, 5]
+
+  0052 0068; [4A04, 5, 46]
+
+  0052 0048; [4A04, 5, 87]
+
+  0074 0068; [4E04, 5, 5]
+
+  0054 0068; [4E04, 5, 46]
+
+  0054 0048; [4E04, 5, 87]'
+:suppressed_contractions: ''
+...

data/resources/collation/tailoring/da.yml

@@ -1,7 +1,7 @@
 ---
 :collator_options:
   :case_first: :upper
-:tailored_table: 0111; [2D, 87, 5]
+:tailored_table: ! '0111; [2D, 87, 5]
 
   0110; [2D, 87, 86]
 
@@ -57,5 +57,6 @@
 
   0041 0061; [5AA1, 5, 48][6, 0, C0]
 
-  0041 0041; [5AA1, 5, 89][6, 0, C0]
+  0041 0041; [5AA1, 5, 89][6, 0, C0]'
 :suppressed_contractions: ''
+...

data/resources/collation/tailoring/de.yml

@@ -2,3 +2,4 @@
 :collator_options: {}
 :tailored_table: ''
 :suppressed_contractions: ''
+...

data/resources/collation/tailoring/el.yml

@@ -2,3 +2,4 @@
 :collator_options: {}
 :tailored_table: ''
 :suppressed_contractions: ''
+...

data/resources/collation/tailoring/en.yml

@@ -2,3 +2,4 @@
 :collator_options: {}
 :tailored_table: ''
 :suppressed_contractions: ''
+...

data/resources/collation/tailoring/es.yml

@@ -1,6 +1,7 @@
 ---
 :collator_options: {}
-:tailored_table: 006E 0303; [4204, 5, 5]
+:tailored_table: ! '006E 0303; [4204, 5, 5]
 
-  004E 0303; [4204, 5, 86]
+  004E 0303; [4204, 5, 86]'
 :suppressed_contractions: ''
+...

data/resources/collation/tailoring/eu.yml

@@ -2,3 +2,4 @@
 :collator_options: {}
 :tailored_table: ''
 :suppressed_contractions: ''
+...

data/resources/collation/tailoring/fa.yml

@@ -1,6 +1,6 @@
 ---
 :collator_options: {}
-:tailored_table: 0650; [0, D3, 5][0, A, C0]
+:tailored_table: ! '0650; [0, D3, 5][0, A, C0]
 
   064F; [0, D3, 5][0, B, C0]
 
@@ -70,5 +70,6 @@
 
   06CD; [6289, 8C, 5]
 
-  06CE; [6289, 8D, 5]
+  06CE; [6289, 8D, 5]'
 :suppressed_contractions: ''
+...

data/resources/collation/tailoring/fi.yml

@@ -1,6 +1,6 @@
 ---
 :collator_options: {}
-:tailored_table: 0111; [2D, 87, 5][4, 5, 5][0, D0, 5][0, 3D, C0]
+:tailored_table: ! '0111; [2D, 87, 5][4, 5, 5][0, D0, 5][0, 3D, C0]
 
   0110; [2D, 87, 86][4, 5, 5][0, D0, 5][0, 3D, C0]
 
@@ -42,5 +42,6 @@
 
   00F8; [5AA1, 87, 5][6, 0, C0]
 
-  00D8; [5AA1, 87, 86][6, 0, C0]
+  00D8; [5AA1, 87, 86][6, 0, C0]'
 :suppressed_contractions: ''
+...

data/resources/collation/tailoring/fil.yml

@@ -1,6 +1,6 @@
 ---
 :collator_options: {}
-:tailored_table: 006E 0303; [4204, 5, 5]
+:tailored_table: ! '006E 0303; [4204, 5, 5]
 
   004E 0303; [4204, 5, 86]
 
@@ -8,5 +8,6 @@
 
   004E 0067; [4205, 5, 46]
 
-  004E 0047; [4205, 5, 87]
+  004E 0047; [4205, 5, 87]'
 :suppressed_contractions: ''
+...

data/resources/collation/tailoring/fr.yml

@@ -2,3 +2,4 @@
 :collator_options: {}
 :tailored_table: ''
 :suppressed_contractions: ''
+...

data/resources/collation/tailoring/ga.yml

@@ -0,0 +1,5 @@
+---
+:collator_options: {}
+:tailored_table: ''
+:suppressed_contractions: ''
+...

data/resources/collation/tailoring/gl.yml

@@ -0,0 +1,5 @@
+---
+:collator_options: {}
+:tailored_table: ''
+:suppressed_contractions: ''
+...

data/resources/collation/tailoring/he.yml

@@ -2,3 +2,4 @@
 :collator_options: {}
 :tailored_table: ''
 :suppressed_contractions: ''
+...

data/resources/collation/tailoring/hi.yml

@@ -1,8 +1,9 @@
 ---
 :collator_options: {}
-:tailored_table: 0902; [680B, 5, 5][4, 0, C0]
+:tailored_table: ! '0902; [680B, 5, 5][4, 0, C0]
 
   0901; [680B, 87, 5][4, 0, C0]
 
-  0903; [680B, 5, 5][5, 0, C0]
+  0903; [680B, 5, 5][5, 0, C0]'
 :suppressed_contractions: ''
+...

data/resources/collation/tailoring/hu.yml

@@ -1,6 +1,6 @@
 ---
 :collator_options: {}
-:tailored_table: 0063 0073; [2C04, 5, 5]
+:tailored_table: ! '0063 0073; [2C04, 5, 5]
 
   0043 0073; [2C04, 5, 47]
 
@@ -122,5 +122,6 @@
 
   005A 007A 0073; [5A04, 5, 48][5A04, 5, 5]
 
-  005A 005A 0053; [5A04, 5, 8A][5A04, 5, 89]
+  005A 005A 0053; [5A04, 5, 8A][5A04, 5, 89]'
 :suppressed_contractions: ''
+...

data/resources/collation/tailoring/id.yml

@@ -2,3 +2,4 @@
 :collator_options: {}
 :tailored_table: ''
 :suppressed_contractions: ''
+...

data/resources/collation/tailoring/it.yml

@@ -2,3 +2,4 @@
 :collator_options: {}
 :tailored_table: ''
 :suppressed_contractions: ''
+...