twitter_cldr 1.5.0 → 1.6.0

data/js/spec/js/calendars/timespan_spec.js

@@ -0,0 +1,91 @@
+// Copyright 2012 Twitter, Inc
+// http://www.apache.org/licenses/LICENSE-2.0
+
+//= require '../../../build/twitter_cldr_en.js'
+
+describe("TimespanFormatter", function() {
+  beforeEach(function() {
+    formatter = new TwitterCldr.TimespanFormatter();
+  });
+
+  describe("#format", function() {
+    it("works for a variety of units for a non-directional timespan", function() {
+      expect(formatter.format(3273932, {
+        unit: "year",
+        direction: "none"
+      })).toEqual('0 years');
+      expect(formatter.format(3273932, {
+        unit: "month"
+        direction: "none",
+      })).toEqual('1 month');
+      expect(formatter.format(3273932, {
+        unit: "week",
+        direction: "none"
+      })).toEqual('5 weeks');
+      expect(formatter.format(3273932, {
+        unit: "day",
+        direction: "none"
+      })).toEqual('38 days');
+      expect(formatter.format(3273932, {
+        unit: "hour",
+        direction: "none"
+      })).toEqual('909 hours');
+      expect(formatter.format(3273932, {
+        unit: "minute",
+        direction: "none"
+      })).toEqual('54566 minutes');
+      expect(formatter.format(3273932, {
+        unit: "second",
+        direction: "none"
+      })).toEqual('3273932 seconds');
+    }),
+
+    it("works for a variety of units in the past", function() {
+      expect(formatter.format(-3273932, {
+        unit: "year"
+      })).toEqual('0 years ago');
+      expect(formatter.format(-3273932, {
+        unit: "month"
+      })).toEqual('1 month ago');
+      expect(formatter.format(-3273932, {
+        unit: "week"
+      })).toEqual('5 weeks ago');
+      expect(formatter.format(-3273932, {
+        unit: "day"
+      })).toEqual('38 days ago');
+      expect(formatter.format(-3273932, {
+        unit: "hour"
+      })).toEqual('909 hours ago');
+      expect(formatter.format(-3273932, {
+        unit: "minute"
+      })).toEqual('54566 minutes ago');
+      expect(formatter.format(-3273932, {
+        unit: "second"
+      })).toEqual('3273932 seconds ago');
+    });
+
+    it("works for a variety of units in the future", function() {
+      expect(formatter.format(3273932, {
+        unit: "year"
+      })).toEqual('In 0 years');
+      expect(formatter.format(3273932, {
+        unit: "month"
+      })).toEqual('In 1 month');
+      expect(formatter.format(3273932, {
+        unit: "week"
+      })).toEqual('In 5 weeks');
+      expect(formatter.format(3273932, {
+        unit: "day"
+      })).toEqual('In 38 days');
+      expect(formatter.format(3273932, {
+        unit: "hour"
+      })).toEqual('In 909 hours');
+      expect(formatter.format(3273932, {
+        unit: "minute"
+      })).toEqual('In 54566 minutes');
+      expect(formatter.format(3273932, {
+        unit: "second"
+      })).toEqual('In 3273932 seconds');
+    });
+  });
+});

data/js/spec/js/plurals/plural_rules_spec.js

@@ -0,0 +1,28 @@
+// Copyright 2012 Twitter, Inc
+// http://www.apache.org/licenses/LICENSE-2.0
+
+//= require '../../../build/twitter_cldr_en.js'
+
+describe("PluralRules", function() {
+  describe("#all", function() {
+    it("returns an array of all English plural rules", function() {
+      expect(TwitterCldr.PluralRules.all()).toEqual(["one", "other"]);
+    });
+  });
+
+  describe("#rule_for", function() {
+    it("returns 'one' for the number 1", function() {
+      expect(TwitterCldr.PluralRules.rule_for(1)).toEqual("one");
+    });
+
+    it("returns 'other' for any number greater than 1", function() {
+      for (var i = 2; i < 10; i ++) {
+        expect(TwitterCldr.PluralRules.rule_for(i)).toEqual("other");
+      }
+    });
+
+    it("returns 'other' for the number 0", function() {
+      expect(TwitterCldr.PluralRules.rule_for(0)).toEqual("other");
+    });
+  });
+});

data/js/spec/js/support/jasmine.yml

@@ -0,0 +1,8 @@
+src_files:
+  - "**/*.*"
+spec_files:
+  - "**/*[Ss]pec.js"
+helpers:
+  - helpers/**/*
+src_dir: ../../build
+spec_dir: ../

data/js/spec/rb/renderers/plurals/plural_rules_compiler_spec.rb

@@ -0,0 +1,52 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require File.join(File.dirname(File.dirname(File.dirname(__FILE__))), "spec_helper")
+
+include TwitterCldr::Js::Renderers::PluralRules
+
+describe PluralRulesCompiler do
+  describe "#rule_to_js" do
+    it "handles a single plural rule" do
+      PluralRulesCompiler.rule_to_js(":other").should == 'function(n) { return "other" }'
+    end
+
+    it "handles a conditional plural rule (eg. English)" do
+      PluralRulesCompiler.rule_to_js("n == 1 ? :one : :other").should == 'function(n) { return (function() { if (n == 1) { return "one" } else { return "other" } })(); }'
+    end
+
+    it "handles an include? call" do
+      PluralRulesCompiler.rule_to_js("[2, 3, 4].include?(n)").should == "function(n) { return [2, 3, 4].indexOf(n) >= 0 }"
+    end
+
+    it "handles the modulus operator" do
+      PluralRulesCompiler.rule_to_js("n % 10").should == "function(n) { return n % 10 }"
+    end
+
+    it "handles < and > operators" do
+      PluralRulesCompiler.rule_to_js("n > 10").should == "function(n) { return n > 10 }"
+      PluralRulesCompiler.rule_to_js("n < 10").should == "function(n) { return n < 10 }"
+    end
+
+    it "handles 'and', 'or', and 'not' operators" do
+      PluralRulesCompiler.rule_to_js("n and n").should == "function(n) { return n && n }"
+      PluralRulesCompiler.rule_to_js("n or n").should == "function(n) { return n || n }"
+      PluralRulesCompiler.rule_to_js("not n").should == "function(n) { return !(n) }"
+    end
+
+    it "compounds include? and the modulus operator" do
+      PluralRulesCompiler.rule_to_js("[2, 3, 4].include?(n % 10)").should == "function(n) { return [2, 3, 4].indexOf(n % 10) >= 0 }"
+    end
+
+    it "compounds include?, modulus, and an if statement" do
+      PluralRulesCompiler.rule_to_js("[2, 3, 4].include?(n % 10) ? :one : :other").should == 'function(n) { return (function() { if ([2, 3, 4].indexOf(n % 10) >= 0) { return "one" } else { return "other" } })(); }'
+    end
+
+    it "chains two if statements (eg. Polish)" do
+      ruby_string = "n == 1 ? :one : [2, 3, 4].include?(n % 10) && ![12, 13, 14].include?(n % 100) && ![22, 23, 24].include?(n % 100) ? :few : :other"
+      PluralRulesCompiler.rule_to_js(ruby_string).should == 'function(n) { return (function() { if (n == 1) { return "one" } else { return (function() { if ([2, 3, 4].indexOf(n % 10) >= 0 && !([12, 13, 14].indexOf(n % 100) >= 0) && !([22, 23, 24].indexOf(n % 100) >= 0)) { return "few" } else { return "other" } })(); } })(); }'
+    end
+  end
+end

data/js/spec/rb/spec_helper.rb

@@ -0,0 +1,13 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'rspec'
+require 'twitter_cldr'
+
+TwitterCldr.require_js
+
+RSpec.configure do |config|
+  config.mock_with :rr
+end

data/lib/twitter_cldr.rb

@@ -23,6 +23,7 @@ module TwitterCldr
   autoload :Formatters,    'twitter_cldr/formatters'
   autoload :Collation,     'twitter_cldr/collation'
   autoload :Normalization, 'twitter_cldr/normalization'
+  autoload :Resources,     'twitter_cldr/resources'
   autoload :Shared,        'twitter_cldr/shared'
   autoload :Tokenizers,    'twitter_cldr/tokenizers'
   autoload :Utils,         'twitter_cldr/utils'
@@ -54,7 +55,7 @@ module TwitterCldr
   class << self
 
     def resources
-      @resources ||= TwitterCldr::Shared::Resources.new
+      @resources ||= TwitterCldr::Resources::Loader.new
     end
 
     def get_locale

data/lib/twitter_cldr/collation.rb

@@ -7,8 +7,9 @@ module TwitterCldr
   module Collation
     autoload :Collator,                  'twitter_cldr/collation/collator'
     autoload :ImplicitCollationElements, 'twitter_cldr/collation/implicit_collation_elements'
-    autoload :SortKey,                   'twitter_cldr/collation/sort_key'
+    autoload :SortKeyBuilder,            'twitter_cldr/collation/sort_key_builder'
     autoload :Trie,                      'twitter_cldr/collation/trie'
     autoload :TrieBuilder,               'twitter_cldr/collation/trie_builder'
+    autoload :TrieWithFallback,          'twitter_cldr/collation/trie_with_fallback'
   end
 end

data/lib/twitter_cldr/collation/collator.rb

@@ -13,52 +13,49 @@ module TwitterCldr
 
       FRACTIONAL_UCA_SHORT_RESOURCE = 'collation/FractionalUCA_SHORT.txt'
 
-      def sort(strings)
-        strings.map{ |s| [s, comparison_key(s)] }.sort{ |a, b| compare_keys(a[1], b[1]) }.map(&:first)
+      attr_accessor :locale
+
+      def initialize(locale = nil)
+        @locale = TwitterCldr.convert_locale(locale) if locale
+        @trie   = load_trie
       end
 
-      def compare(string_a, string_b)
-        string_a == string_b ? 0 : compare_keys(comparison_key(string_a), comparison_key(string_b))
+      def sort(strings)
+        strings.map{ |s| [s, get_sort_key(s)] }.sort{ |a, b| a[1] <=> b[1] }.map(&:first)
       end
 
-      def sort_key(string_or_code_points)
-        sort_key_for_code_points(get_code_points(string_or_code_points))
+      def sort!(strings)
+        sort_keys = Hash.new { |hash, string| hash[string] = get_sort_key(string) }
+        strings.replace(strings.sort_by { |s| sort_keys[s] })
       end
 
-      def trie
-        @trie ||= self.class.trie
+      def compare(string_a, string_b)
+        string_a == string_b ? 0 : get_sort_key(string_a) <=> get_sort_key(string_b)
       end
 
-      def self.trie
-        @trie ||= TwitterCldr::Collation::TrieBuilder.load_trie(FRACTIONAL_UCA_SHORT_RESOURCE)
+      def get_sort_key(string_or_code_points)
+        TwitterCldr::Collation::SortKeyBuilder.build(get_collation_elements(string_or_code_points))
       end
 
-      private
+      def get_collation_elements(string_or_code_points)
+        integer_code_points = get_normalized_code_points(string_or_code_points)
 
-      def comparison_key(string)
-        code_points = TwitterCldr::Utils::CodePoints.from_string(string)
-        { :code_points => code_points, :sort_key => sort_key(code_points) }
+        result = []
+        result.concat(code_point_collation_elements(integer_code_points)) until integer_code_points.empty?
+        result
       end
 
-      def compare_keys(a, b)
-        (a[:sort_key] <=> b[:sort_key]).nonzero? || get_integer_code_points(a[:code_points]) <=> get_integer_code_points(b[:code_points])
-      end
+      private
 
-      def sort_key_for_code_points(integer_code_points)
-        TwitterCldr::Collation::SortKey.build(get_collation_elements(integer_code_points))
+      def load_trie
+        @locale ? self.class.tailored_fce_trie(@locale) : self.class.default_fce_trie
       end
 
       def get_integer_code_points(code_points)
         code_points.map { |code_point| code_point.to_i(16) }
       end
 
-      def get_collation_elements(integer_code_points)
-        result = []
-        result.concat(code_point_collation_elements(integer_code_points)) until integer_code_points.empty?
-        result
-      end
-
-      def get_code_points(str_or_code_points)
+      def get_normalized_code_points(str_or_code_points)
         code_points = str_or_code_points.is_a?(String) ? TwitterCldr::Utils::CodePoints.from_string(str_or_code_points) : str_or_code_points
 
         # Normalization makes the collation process significantly slower (like seven times slower on the UCA
@@ -89,7 +86,7 @@ module TwitterCldr
       #
       def explicit_collation_elements(integer_code_points)
         # find the longest prefix in the trie
-        collation_elements, suffixes, prefix_size = trie.find_prefix(integer_code_points)
+        collation_elements, prefix_size, suffixes = @trie.find_prefix(integer_code_points)
 
         return unless collation_elements
 
@@ -101,9 +98,6 @@ module TwitterCldr
         used_combining_classes = {}
 
         while non_starter_pos < integer_code_points.size && !suffixes.empty?
-          # create a trie from a hash of suffixes available for the chosen prefix
-          subtrie = TwitterCldr::Collation::Trie.new(suffixes)
-
           # get next code point (possibly non-starter)
           non_starter_code_point = integer_code_points[non_starter_pos]
           combining_class        = TwitterCldr::Normalization::Base.combining_class_for(non_starter_code_point.to_s(16))
@@ -115,7 +109,7 @@ module TwitterCldr
 
           # Try to find collation elements for [prefix + non-starter] code points sequence. As the subtrie contains
           # suffixes (without prefix) we pass only non-starter itself.
-          new_collation_elements, new_suffixes = subtrie.find_prefix([non_starter_code_point]).first(2)
+          new_collation_elements, _, new_suffixes = suffixes.find_prefix([non_starter_code_point])
 
           if new_collation_elements
             # non-starter with a collation elements sequence corresponding to [prefix + non-starter] accepted
@@ -137,6 +131,30 @@ module TwitterCldr
         TwitterCldr::Collation::ImplicitCollationElements.for_code_point(integer_code_points.shift)
       end
 
+      class << self
+
+        # Loads and memoizes the default Fractional Collation Elements trie.
+        #
+        def default_fce_trie
+          @default_fce_trie ||= TwitterCldr::Collation::TrieBuilder.load_trie(FRACTIONAL_UCA_SHORT_RESOURCE).lock
+        end
+
+        def tailored_fce_trie(locale)
+          tailored_fce_tries_cache[locale]
+        end
+
+        private
+
+        def tailored_fce_tries_cache
+          @tailored_fce_tries_cache ||= Hash.new { |hash, locale| hash[locale] = load_tailored_trie(locale) }
+        end
+
+        def load_tailored_trie(locale)
+          TwitterCldr::Collation::TrieBuilder.load_tailored_trie(locale, default_fce_trie).lock
+        end
+
+      end
+
     end
 
   end

data/lib/twitter_cldr/collation/{sort_key.rb → sort_key_builder.rb}

@@ -6,15 +6,23 @@
 module TwitterCldr
   module Collation
 
-    # SortKey builds a collation sort key from an array of collation elements.
+    # SortKeyBuilder builds a collation sort key from an array of collation elements.
     #
-    class SortKey
+    # Weights compression algorithms for every level are described in http://source.icu-project.org/repos/icu/icuhtml/trunk/design/collation/ICU_collation_design.htm
+    #
+    class SortKeyBuilder
 
       PRIMARY_LEVEL, SECONDARY_LEVEL, TERTIARY_LEVEL = 0, 1, 2
 
       LEVEL_SEPARATOR = 1 # separate levels in a sort key '01' bytes
 
-      TERTIARY_LEVEL_MASK = 0x3F # mask for removing case bits from tertiary weight ('CC' bits in 'CC00 0000')
+      TERTIARY_LEVEL_MASK = 0x3F # mask for removing case bits or continuation flag from a tertiary weight
+
+      PRIMARY_BYTE_MIN = 0x3
+      PRIMARY_BYTE_MAX = 0xFF
+
+      MIN_NON_LATIN_PRIMARY = 0x5B
+      MAX_REGULAR_PRIMARY   = 0x7A
 
       attr_reader :collation_elements
 
@@ -56,11 +64,30 @@ module TwitterCldr
       end
 
       def append_primary_bytes
+        @last_leading_byte = nil
+
         @collation_elements.each do |collation_element|
-          append_weight(level_weight(collation_element, PRIMARY_LEVEL))
+          bytes = fixnum_to_bytes_array(level_weight(collation_element, PRIMARY_LEVEL))
+
+          unless bytes.empty?
+            leading_byte = bytes.shift
+
+            if leading_byte != @last_leading_byte
+              @bytes_array << (leading_byte < @last_leading_byte ? PRIMARY_BYTE_MIN : PRIMARY_BYTE_MAX) if @last_leading_byte
+              @bytes_array << leading_byte
+
+              @last_leading_byte = !bytes.empty? && compressible_primary?(leading_byte) ? leading_byte : nil
+            end
+
+            @bytes_array.concat(bytes)
+          end
         end
       end
 
+      def compressible_primary?(leading_byte)
+        (MIN_NON_LATIN_PRIMARY..MAX_REGULAR_PRIMARY).include?(leading_byte)
+      end
+
       def append_secondary_bytes
         @bytes_array << LEVEL_SEPARATOR
 
@@ -136,10 +163,6 @@ module TwitterCldr
         level_weight(collation_element, TERTIARY_LEVEL) & TERTIARY_LEVEL_MASK
       end
 
-      def append_weight(weight)
-        @bytes_array.concat(fixnum_to_bytes_array(weight))
-      end
-
       def level_weight(collation_element, level)
         collation_element[level] || 0
       end

data/lib/twitter_cldr/collation/trie.rb

@@ -17,54 +17,146 @@ module TwitterCldr
     class Trie
 
       # Initializes a new trie. If `trie_hash` value is passed it's used as the initial data for the trie. Usually,
-      # `trie_hash` is extracted from other trie and represents its sub-trie.
+      # `trie_hash` is extracted from other trie and represents its subtrie.
       #
-      def initialize(trie_hash = {})
-        @root = [nil, trie_hash]
+      def initialize(root = Node.new)
+        @root = root
+        @locked = false
+      end
+
+      def lock
+        @locked = true
+        self
+      end
+
+      def locked?
+        @locked
+      end
+
+      def starters
+        @root.keys
+      end
+
+      def each_starting_with(starter, &block)
+        starting_node = @root.child(starter)
+        each_pair(starting_node, [starter], &block) if starting_node
+      end
+
+      def empty?
+        !@root.has_children?
       end
 
       def add(key, value)
-        final = key.inject(@root) do |node, key_element|
-          node[1][key_element] ||= [nil, {}]
-        end
+        store(key, value, false)
+      end
 
-        final[0] = value
+      def set(key, value)
+        store(key, value)
       end
 
       def get(key)
         final = key.inject(@root) do |node, key_element|
-          subtree = node[1][key_element]
-          return unless subtree
-          subtree
+          return unless node
+          node.child(key_element)
         end
 
-        final[0]
+        final && final.value
       end
 
       # Finds the longest substring of the `key` that matches, as a key, a node in the trie.
       #
       # Returns a three elements array:
       #
-      #   1. value in the last node that was visited
-      #   2. sub-trie of this node (as a hash)
-      #   3. size of the `key` prefix that matches this node
+      #   1. value in the last node that was visited and has non-nil value
+      #   2. size of the `key` prefix that matches this node
+      #   3. subtrie for which that node is a root
       #
       def find_prefix(key)
-        prefix_size = 0
-        node = @root
+        last_prefix_size = 0
+        last_with_value  = @root
+
+        key.each_with_index.inject(@root) do |node, (key_element, index)|
+          child = node.child(key_element)
+
+          break unless child
+
+          if child.value
+            last_prefix_size = index + 1
+            last_with_value  = child
+          end
+
+          child
+        end
+
+        [last_with_value.value, last_prefix_size, last_with_value.to_trie]
+      end
+
+      def to_hash
+        @root.subtrie_hash
+      end
+
+      alias inspect to_s # to prevent printing of a possibly huge children list in the IRB
+
+      private
 
-        key.each do |key_element|
-          subtree = node[1][key_element]
+      def store(key, value, override = true)
+        raise RuntimeError, "can't store value in a locked trie" if locked?
+
+        final = key.inject(@root) do |node, key_element|
+          node.child(key_element) || node.set_child(key_element, Node.new)
+        end
+
+        final.value = value unless final.value && !override
+      end
+
+      def each_pair(node, key, &block)
+        yield [key, node.value] if node.value
+
+        node.each_key_and_child do |key_element, child|
+          each_pair(child, key + [key_element], &block)
+        end
+      end
+
+      class Node
+
+        attr_accessor :value
+
+        def initialize(value = nil, children = {})
+          @value    = value
+          @children = children
+        end
+
+        def child(key)
+          @children[key]
+        end
+
+        def set_child(key, child)
+          @children[key] = child
+        end
+
+        def has_children?
+          !@children.empty?
+        end
+
+        def each_key_and_child(&block)
+          @children.each(&block)
+        end
+
+        def keys
+          @children.keys
+        end
+
+        def to_trie
+          Trie.new(self.class.new(nil, @children)).lock
+        end
 
-          if subtree
-            prefix_size += 1
-            node = subtree
-          else
-            break
+        def subtrie_hash
+          @children.inject({}) do |memo, (key, child)|
+            memo[key] = [child.value, child.subtrie_hash]
+            memo
           end
         end
 
-        node + [prefix_size]
       end
 
     end