twitter_cldr 1.6.0 → 1.6.1

data/Gemfile

@@ -9,6 +9,7 @@ end
 group :development do
   gem 'mustache', '~> 0.99.4'
   gem 'nokogiri'
+  gem 'ruby_parser', '~> 2.3.1'
 
   platform :ruby do
     gem 'therubyracer',  '~> 0.9.10'

data/History.txt

@@ -1,3 +1,8 @@
+=== 1.6.1
+
+* Added case-first collation element tailoring support for languages like Danish.
+* Included a missing development dependency (ruby_parser).
+
 === 1.6.0
 
 * Added locale-aware collation via fractional collation element tailoring.

data/js/lib/twitter_cldr_js.rb

@@ -10,6 +10,7 @@ require 'uglifier'
 require 'jasmine-headless-webkit'
 require 'coffee-script'
 require 'json'
+require 'ruby_parser'
 
 require 'compiler'
 require 'renderers/bundle'
@@ -58,6 +59,7 @@ module TwitterCldr
 
     def self.make(options = {})
       # clean dir, then build js
+      FileUtils.mkdir_p(build_dir)
       FileUtils.rm_rf(Dir.glob(File.join(build_dir, "**")))
       build(options)
       build(options.merge({ :minify => true }))

data/lib/twitter_cldr/collation/collator.rb

@@ -16,8 +16,9 @@ module TwitterCldr
       attr_accessor :locale
 
       def initialize(locale = nil)
-        @locale = TwitterCldr.convert_locale(locale) if locale
-        @trie   = load_trie
+        @locale  = TwitterCldr.convert_locale(locale) if locale
+        @options = tailoring_options
+        @trie    = load_trie
       end
 
       def sort(strings)
@@ -34,7 +35,7 @@ module TwitterCldr
       end
 
       def get_sort_key(string_or_code_points)
-        TwitterCldr::Collation::SortKeyBuilder.build(get_collation_elements(string_or_code_points))
+        TwitterCldr::Collation::SortKeyBuilder.build(get_collation_elements(string_or_code_points), @options[:case_first])
       end
 
       def get_collation_elements(string_or_code_points)
@@ -47,6 +48,10 @@ module TwitterCldr
 
       private
 
+      def tailoring_options
+        @locale ? TwitterCldr::Collation::TrieBuilder.tailoring_data(@locale)[:collator_options] : {}
+      end
+
       def load_trie
         @locale ? self.class.tailored_fce_trie(@locale) : self.class.default_fce_trie
       end

data/lib/twitter_cldr/collation/sort_key_builder.rb

@@ -8,7 +8,8 @@ module TwitterCldr
 
     # SortKeyBuilder builds a collation sort key from an array of collation elements.
     #
-    # Weights compression algorithms for every level are described in http://source.icu-project.org/repos/icu/icuhtml/trunk/design/collation/ICU_collation_design.htm
+    # Weights compression algorithms for every level are described in
+    # http://source.icu-project.org/repos/icu/icuhtml/trunk/design/collation/ICU_collation_design.htm
     #
     class SortKeyBuilder
 
@@ -16,35 +17,36 @@ module TwitterCldr
 
       LEVEL_SEPARATOR = 1 # separate levels in a sort key '01' bytes
 
-      TERTIARY_LEVEL_MASK = 0x3F # mask for removing case bits or continuation flag from a tertiary weight
+      VALID_CASE_FIRST_OPTIONS = [nil, :lower, :upper]
 
-      PRIMARY_BYTE_MIN = 0x3
-      PRIMARY_BYTE_MAX = 0xFF
-
-      MIN_NON_LATIN_PRIMARY = 0x5B
-      MAX_REGULAR_PRIMARY   = 0x7A
-
-      attr_reader :collation_elements
+      attr_reader :collation_elements, :case_first
 
       # Returns a sort key as an array of bytes.
       #
       # Arguments:
       #
       #   collation_elements - an array of collation elements, represented as arrays of integer weights.
+      #   case_first         - case-first sorting order setting.
       #
       # An instance of the class is created only to prevent passing of @collation_elements and @bytes_array from one
       # method into another while forming the sort key.
       #
-      def self.build(collation_elements)
-        new(collation_elements).bytes_array
+      def self.build(collation_elements, case_first = nil)
+        new(collation_elements, case_first).bytes_array
       end
 
       # Arguments:
       #
       #   collation_elements - an array of collation elements, represented as arrays of integer weights.
+      #   case_first         - optional case-first sorting order setting: :upper, :lower, nil (discard case bits).
       #
-      def initialize(collation_elements)
+      def initialize(collation_elements, case_first = nil)
+        raise ArgumentError, "invalid case-first options '#{case_first.inspect}'" unless VALID_CASE_FIRST_OPTIONS.include?(case_first)
+
         @collation_elements = collation_elements
+        @case_first         = case_first
+
+        init_tertiary_constants
       end
 
       def bytes_array
@@ -115,7 +117,14 @@ module TwitterCldr
         end
 
         # append compressed trailing common bytes
-        append_common_bytes(TERTIARY_BOTTOM, TERTIARY_BOTTOM_COUNT, false) if @common_count > 0
+        if @common_count > 0
+          if @tertiary_common == TERTIARY_BOTTOM_NORMAL
+            append_common_bytes(@tertiary_bottom, @tertiary_bottom_count, false)
+          else
+            append_common_bytes(@tertiary_top, @tertiary_top_count, true)
+            @bytes_array[-1] -= 1 # make @bytes_array[-1] = boundary - @common_count (for compatibility with ICU)
+          end
+        end
       end
 
       def append_secondary_byte(secondary)
@@ -127,11 +136,16 @@ module TwitterCldr
       end
 
       def append_tertiary_byte(tertiary)
-        if tertiary == TERTIARY_COMMON
+        if tertiary == @tertiary_common
           @common_count += 1
         else
-          tertiary += TERTIARY_TOP_ADDITION if tertiary > TERTIARY_COMMON # create a gap above TERTIARY_COMMON
-          append_with_common_bytes(tertiary, TERTIARY_COMMON_SPACE)
+          if @tertiary_common == TERTIARY_COMMON_NORMAL && @tertiary_common < tertiary
+            tertiary += @tertiary_addition
+          elsif @tertiary_common == TERTIARY_COMMON_UPPER_FIRST && tertiary <= @tertiary_common
+            tertiary -= @tertiary_addition
+          end
+
+          append_with_common_bytes(tertiary, @tertiary_common_space)
         end
       end
 
@@ -160,7 +174,13 @@ module TwitterCldr
       end
 
       def tertiary_weight(collation_element)
-        level_weight(collation_element, TERTIARY_LEVEL) & TERTIARY_LEVEL_MASK
+        weight = level_weight(collation_element, TERTIARY_LEVEL)
+
+        if continuation?(weight)
+          remove_continuation_bits(weight)
+        else
+          (weight & @tertiary_mask) ^ @case_switch
+        end
       end
 
       def level_weight(collation_element, level)
@@ -178,6 +198,60 @@ module TwitterCldr
         bytes
       end
 
+      def continuation?(weight)
+        weight & CASE_BITS_MASK == CASE_BITS_MASK
+      end
+
+      def remove_continuation_bits(weight)
+        weight & REMOVE_CASE_MASK
+      end
+
+      def init_tertiary_constants
+        @case_switch = @case_first == :upper ? CASE_SWITCH : NO_CASE_SWITCH
+
+        if @case_first
+          @tertiary_mask     = KEEP_CASE_MASK
+          @tertiary_addition = TERTIARY_ADDITION_CASE_FIRST
+
+          if @case_first == :upper
+            @tertiary_common = TERTIARY_COMMON_UPPER_FIRST
+            @tertiary_top    = TERTIARY_TOP_UPPER_FIRST
+            @tertiary_bottom = TERTIARY_BOTTOM_UPPER_FIRST
+          else # @case_first == :lower
+            @tertiary_common = TERTIARY_COMMON_NORMAL
+            @tertiary_top    = TERTIARY_TOP_LOWER_FIRST
+            @tertiary_bottom = TERTIARY_BOTTOM_LOWER_FIRST
+          end
+        else
+          @tertiary_mask     = REMOVE_CASE_MASK
+          @tertiary_addition = TERTIARY_ADDITION_NORMAL
+
+          @tertiary_common = TERTIARY_COMMON_NORMAL
+          @tertiary_top    = TERTIARY_TOP_NORMAL
+          @tertiary_bottom = TERTIARY_BOTTOM_NORMAL
+        end
+
+        total_tertiary_count   = @tertiary_top - @tertiary_bottom - 1
+        @tertiary_top_count    = (TERTIARY_PROPORTION * total_tertiary_count).to_i
+        @tertiary_bottom_count = total_tertiary_count - @tertiary_top_count
+
+        @tertiary_common_space = {
+            :common       => @tertiary_common,
+            :bottom       => @tertiary_bottom,
+            :bottom_count => @tertiary_bottom_count,
+            :top          => @tertiary_top,
+            :top_count    => @tertiary_top_count
+        }
+      end
+
+      # Primary level compression constants
+
+      PRIMARY_BYTE_MIN = 0x3
+      PRIMARY_BYTE_MAX = 0xFF
+
+      MIN_NON_LATIN_PRIMARY = 0x5B
+      MAX_REGULAR_PRIMARY   = 0x7A
+
       # Secondary level compression constants
 
       SECONDARY_BOTTOM       = 0x05
@@ -198,23 +272,33 @@ module TwitterCldr
 
       # Tertiary level compression constants
 
-      TERTIARY_TOP_ADDITION = 0x80
-
-      TERTIARY_BOTTOM       = 0x05
-      TERTIARY_TOP          = 0x85
-      TERTIARY_PROPORTION   = 0.667
-      TERTIARY_COMMON       = TERTIARY_BOTTOM
-      TERTIARY_TOTAL_COUNT  = TERTIARY_TOP - TERTIARY_BOTTOM - 1
-      TERTIARY_TOP_COUNT    = (TERTIARY_PROPORTION * TERTIARY_TOTAL_COUNT).to_i
-      TERTIARY_BOTTOM_COUNT = TERTIARY_TOTAL_COUNT - TERTIARY_TOP_COUNT
-
-      TERTIARY_COMMON_SPACE = {
-          :common       => TERTIARY_COMMON,
-          :bottom       => TERTIARY_BOTTOM,
-          :bottom_count => TERTIARY_BOTTOM_COUNT,
-          :top          => TERTIARY_TOP,
-          :top_count    => TERTIARY_TOP_COUNT
-      }
+      REMOVE_CASE_MASK = 0x3F
+      KEEP_CASE_MASK   = 0xFF
+
+      CASE_BITS_MASK = 0xC0
+
+      CASE_SWITCH    = 0xC0
+      NO_CASE_SWITCH = 0
+
+      TERTIARY_ADDITION_NORMAL     = 0x80
+      TERTIARY_ADDITION_CASE_FIRST = 0x40
+
+      TERTIARY_PROPORTION = 0.667
+
+      # Normal (case-first disabled)
+      TERTIARY_BOTTOM_NORMAL = 0x05
+      TERTIARY_TOP_NORMAL    = 0x85
+      TERTIARY_COMMON_NORMAL = TERTIARY_BOTTOM_NORMAL
+
+      # Lower first
+      TERTIARY_BOTTOM_LOWER_FIRST = TERTIARY_BOTTOM_NORMAL
+      TERTIARY_TOP_LOWER_FIRST    = 0x45
+      TERTIARY_COMMON_LOWER_FIRST = TERTIARY_BOTTOM_LOWER_FIRST
+
+      # Upper first
+      TERTIARY_BOTTOM_UPPER_FIRST = 0x86
+      TERTIARY_TOP_UPPER_FIRST    = 0xC5
+      TERTIARY_COMMON_UPPER_FIRST = TERTIARY_TOP_UPPER_FIRST
 
     end
 

data/lib/twitter_cldr/collation/trie_builder.rb

@@ -20,7 +20,7 @@ module TwitterCldr
         end
 
         def load_tailored_trie(locale, fallback)
-          build_tailored_trie(TwitterCldr.get_resource(:collation, :tailoring, locale), fallback)
+          build_tailored_trie(tailoring_data(locale), fallback)
         end
 
         def parse_trie(table, trie = TwitterCldr::Collation::Trie.new)
@@ -31,6 +31,10 @@ module TwitterCldr
           trie
         end
 
+        def tailoring_data(locale)
+          TwitterCldr.get_resource(:collation, :tailoring, locale)
+        end
+
         private
 
         def load_resource(resource)

data/lib/twitter_cldr/resources/import/tailoring.rb

@@ -33,7 +33,7 @@ module TwitterCldr
             :pt => :root
         }
 
-        EMPTY_TAILORING_DATA = { 'tailored_table' => '', 'suppressed_contractions' => '' }
+        EMPTY_TAILORING_DATA = { 'collator_options' => {}, 'tailored_table' => '', 'suppressed_contractions' => '' }
 
         class ImportError < RuntimeError; end
 
@@ -98,6 +98,7 @@ module TwitterCldr
           standard_tailoring = collations.at_xpath('collation[@type="standard"]')
 
           {
+              'collator_options'        => parse_collator_options(standard_tailoring),
               'tailored_table'          => parse_tailorings(standard_tailoring, locale),
               'suppressed_contractions' => parse_suppressed_contractions(standard_tailoring)
           }
@@ -147,11 +148,19 @@ module TwitterCldr
         end
 
         def parse_suppressed_contractions(data)
-          return '' unless data
+          node = data && data.at_xpath('suppress_contractions')
+          node ? Java::ComIbmIcuText::UnicodeSet.to_array(Java::ComIbmIcuText::UnicodeSet.new(node.text)).to_a.join : ''
+        end
+
+        def parse_collator_options(data)
+          options = {}
+
+          if data
+            case_first_setting = data.at_xpath('settings[@caseFirst]')
+            options['case_first'] = case_first_setting.attr('caseFirst').to_sym if case_first_setting
+          end
 
-          Array(data.xpath('suppress_contractions')).map do |contractions|
-            Java::ComIbmIcuText::UnicodeSet.to_array(Java::ComIbmIcuText::UnicodeSet.new(contractions.text)).to_a
-          end.flatten.join
+          options
         end
 
         def validate_tailoring_rule(rule)

data/lib/twitter_cldr/version.rb

@@ -4,5 +4,5 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 
 module TwitterCldr
-  VERSION = "1.6.0"
+  VERSION = "1.6.1"
 end

data/resources/collation/tailoring/af.yml

@@ -1,3 +1,4 @@
 ---
+collator_options: {}
 tailored_table: 0149; [41, 5, 10]
 suppressed_contractions: ''

data/resources/collation/tailoring/ar.yml

@@ -1,4 +1,5 @@
 ---
+collator_options: {}
 tailored_table: 0629; [6144, 87, 5]
 
   FE94; [6144, 87, 6]

data/resources/collation/tailoring/ca.yml

@@ -1,4 +1,5 @@
 ---
+collator_options: {}
 tailored_table: 0140; [3D, 87, 5]
 
   006C 00B7; [3D, 87, 5]

data/resources/collation/tailoring/cs.yml

@@ -1,4 +1,5 @@
 ---
+collator_options: {}
 tailored_table: 0063 030C; [2C04, 5, 5]
 
   0043 030C; [2C04, 5, 86]

data/resources/collation/tailoring/da.yml

@@ -1,4 +1,6 @@
 ---
+collator_options:
+  case_first: :upper
 tailored_table: 0111; [2D, 87, 5]
 
   0110; [2D, 87, 86]

data/resources/collation/tailoring/de.yml

@@ -1,3 +1,4 @@
 ---
+collator_options: {}
 tailored_table: ''
 suppressed_contractions: ''

data/resources/collation/tailoring/el.yml

@@ -1,3 +1,4 @@
 ---
+collator_options: {}
 tailored_table: ''
 suppressed_contractions: ''

data/resources/collation/tailoring/en.yml

@@ -1,3 +1,4 @@
 ---
+collator_options: {}
 tailored_table: ''
 suppressed_contractions: ''

data/resources/collation/tailoring/es.yml

@@ -1,4 +1,5 @@
 ---
+collator_options: {}
 tailored_table: 006E 0303; [4204, 5, 5]
 
   004E 0303; [4204, 5, 86]

data/resources/collation/tailoring/eu.yml

@@ -1,3 +1,4 @@
 ---
+collator_options: {}
 tailored_table: ''
 suppressed_contractions: ''

data/resources/collation/tailoring/fa.yml

@@ -1,4 +1,5 @@
 ---
+collator_options: {}
 tailored_table: 0650; [0, D3, 5][0, A, C0]
 
   064F; [0, D3, 5][0, B, C0]

data/resources/collation/tailoring/fi.yml

@@ -1,4 +1,5 @@
 ---
+collator_options: {}
 tailored_table: 00F0; [2D, 5, 9][0, DB, 9][0, A9, C0]
 
   00D0; [2D, 5, 93][0, DB, 9][0, A9, C0]

data/resources/collation/tailoring/fil.yml

@@ -1,4 +1,5 @@
 ---
+collator_options: {}
 tailored_table: 006E 0303; [4204, 5, 5]
 
   004E 0303; [4204, 5, 86]

data/resources/collation/tailoring/fr.yml

@@ -1,3 +1,4 @@
 ---
+collator_options: {}
 tailored_table: ''
 suppressed_contractions: ''

data/resources/collation/tailoring/he.yml

@@ -1,3 +1,4 @@
 ---
+collator_options: {}
 tailored_table: ''
 suppressed_contractions: ''

data/resources/collation/tailoring/hi.yml

@@ -1,4 +1,5 @@
 ---
+collator_options: {}
 tailored_table: 0902; [680B, 5, 5][4, 0, C0]
 
   0901; [680B, 87, 5][4, 0, C0]

data/resources/collation/tailoring/hu.yml

@@ -1,4 +1,5 @@
 ---
+collator_options: {}
 tailored_table: 0063 0073; [2C04, 5, 5]
 
   0043 0073; [2C04, 5, 47]

data/resources/collation/tailoring/id.yml

@@ -1,3 +1,4 @@
 ---
+collator_options: {}
 tailored_table: ''
 suppressed_contractions: ''

data/resources/collation/tailoring/it.yml

@@ -1,3 +1,4 @@
 ---
+collator_options: {}
 tailored_table: ''
 suppressed_contractions: ''

data/resources/collation/tailoring/ja.yml

@@ -1,4 +1,5 @@
 ---
+collator_options: {}
 tailored_table: 309D; [E29, 5, 5]
 
   30A1 30FC; [790A, 5, 1D][790A, 5, 1C]

data/resources/collation/tailoring/ko.yml

@@ -1,4 +1,5 @@
 ---
+collator_options: {}
 tailored_table: 4F3D; [780A, FE, 5][0, E, C0][7834, 5, 5]
 
   4F73; [780A, FE, 5][0, F, C0][7834, 5, 5]

data/resources/collation/tailoring/ms.yml

@@ -1,3 +1,4 @@
 ---
+collator_options: {}
 tailored_table: ''
 suppressed_contractions: ''

data/resources/collation/tailoring/nb.yml

@@ -1,4 +1,5 @@
 ---
+collator_options: {}
 tailored_table: 0111; [2D, 87, 5]
 
   0110; [2D, 87, 86]

data/resources/collation/tailoring/nl.yml

@@ -1,3 +1,4 @@
 ---
+collator_options: {}
 tailored_table: ''
 suppressed_contractions: ''

data/resources/collation/tailoring/pl.yml

@@ -1,4 +1,5 @@
 ---
+collator_options: {}
 tailored_table: 0061 0328; [2804, 5, 5]
 
   0041 0328; [2804, 5, 86]

data/resources/collation/tailoring/pt.yml

@@ -1,3 +1,4 @@
 ---
+collator_options: {}
 tailored_table: ''
 suppressed_contractions: ''

data/resources/collation/tailoring/ru.yml

@@ -1,3 +1,4 @@
 ---
+collator_options: {}
 tailored_table: ''
 suppressed_contractions: ІАГЕЖЗКОУЧЫЭагежзкоучыэіѴѵӘәӨө

data/resources/collation/tailoring/sv.yml

@@ -1,4 +1,5 @@
 ---
+collator_options: {}
 tailored_table: 0111; [2D, 87, 5]
 
   0110; [2D, 87, 86]

data/resources/collation/tailoring/th.yml

@@ -1,4 +1,5 @@
 ---
+collator_options: {}
 tailored_table: 0E2F; [C3E, 5, 5]
 
   0E46; [C42, 5, 5]

data/resources/collation/tailoring/tr.yml

@@ -1,4 +1,5 @@
 ---
+collator_options: {}
 tailored_table: 0063 0327; [2C04, 5, 5]
 
   0043 0327; [2C04, 5, 86]

data/resources/collation/tailoring/uk.yml

@@ -1,4 +1,5 @@
 ---
+collator_options: {}
 tailored_table: 0491; [5C1B, 5, 5]
 
   0490; [5C1B, 5, 86]

data/resources/collation/tailoring/ur.yml

@@ -1,4 +1,5 @@
 ---
+collator_options: {}
 tailored_table: 0627 0654; [6126, 87, 5]
 
   0627 0653; [6127, 5, 5][4, 0, C0]

data/resources/collation/tailoring/zh-Hant.yml

@@ -1,3 +1,4 @@
 ---
+collator_options: {}
 tailored_table: ''
 suppressed_contractions: ''

data/resources/collation/tailoring/zh.yml

@@ -1,4 +1,5 @@
 ---
+collator_options: {}
 tailored_table: 0061 0304; [27, 4, 5][0, 4, C0]
 
   0041 0304; [27, 4, 86][0, 4, C0]