twitter_cldr_js 2.3.2 → 2.4.0

data/lib/twitter_cldr/js/compiler.rb

@@ -16,7 +16,7 @@ module TwitterCldr
 
       def initialize(options = {})
         @locales = options[:locales] || TwitterCldr.supported_locales
-        @features = options[:features] || renderers.keys
+        @features = options[:features] || implementation_renderers.keys
         @prerender = options[:prerender].nil? ? true : options[:prerender]
         @source_map = options[:source_map]
       end
@@ -28,7 +28,7 @@ module TwitterCldr
           contents = ""
 
           @features.each do |feature|
-            renderer_const = renderers[feature]
+            renderer_const = implementation_renderers[feature]
             contents << renderer_const.new(:locale => locale, :prerender => @prerender).render if renderer_const
           end
 
@@ -58,8 +58,8 @@ module TwitterCldr
 
       private
 
-      def renderers
-        @renderers ||= {
+      def implementation_renderers
+        @implementation_renderers ||= {
           :plural_rules                    => TwitterCldr::Js::Renderers::PluralRules::PluralRulesRenderer,
           :timespan                        => TwitterCldr::Js::Renderers::Calendars::TimespanRenderer,
           :datetime                        => TwitterCldr::Js::Renderers::Calendars::DateTimeRenderer,
@@ -69,11 +69,32 @@ module TwitterCldr
           :lists                           => TwitterCldr::Js::Renderers::Shared::ListRenderer,
           :bidi                            => TwitterCldr::Js::Renderers::Shared::BidiRenderer,
           :calendar                        => TwitterCldr::Js::Renderers::Shared::CalendarRenderer,
+          :code_point                      => TwitterCldr::Js::Renderers::Shared::CodePointRenderer,
           :phone_codes                     => TwitterCldr::Js::Renderers::Shared::PhoneCodesRenderer,
           :postal_codes                    => TwitterCldr::Js::Renderers::Shared::PostalCodesRenderer,
           :languages                       => TwitterCldr::Js::Renderers::Shared::LanguagesRenderer,
-          :number_parser                   => TwitterCldr::Js::Renderers::Parsers::NumberParser,
+          :unicode_regex                   => TwitterCldr::Js::Renderers::Shared::UnicodeRegexRenderer,
+          :break_iterator                  => TwitterCldr::Js::Renderers::Shared::BreakIteratorRenderer,
           :territories_containment         => TwitterCldr::Js::Renderers::Shared::TerritoriesContainmentRenderer,
+          :number_parser                   => TwitterCldr::Js::Renderers::Parsers::NumberParser,
+          :component                       => TwitterCldr::Js::Renderers::Parsers::ComponentRenderer,
+          :literal                         => TwitterCldr::Js::Renderers::Parsers::LiteralRenderer,
+          :unicode_string                  => TwitterCldr::Js::Renderers::Parsers::UnicodeStringRenderer,
+          :character_class                 => TwitterCldr::Js::Renderers::Parsers::CharacterClassRenderer,
+          :character_range                 => TwitterCldr::Js::Renderers::Parsers::CharacterRangeRenderer,
+          :character_set                   => TwitterCldr::Js::Renderers::Parsers::CharacterSetRenderer,
+          :symbol_table                    => TwitterCldr::Js::Renderers::Parsers::SymbolTableRenderer,
+          :parser                          => TwitterCldr::Js::Renderers::Parsers::ParserRenderer,
+          :segmentation_parser             => TwitterCldr::Js::Renderers::Parsers::SegmentationParserRenderer,
+          :unicode_regex_parser            => TwitterCldr::Js::Renderers::Parsers::UnicodeRegexParserRenderer,
+          :token                           => TwitterCldr::Js::Renderers::Tokenizers::TokenRenderer,
+          :composite_token                 => TwitterCldr::Js::Renderers::Tokenizers::CompositeTokenRenderer,
+          :tokenizer                       => TwitterCldr::Js::Renderers::Tokenizers::TokenizerRenderer,
+          :segmentation_tokenizer          => TwitterCldr::Js::Renderers::Tokenizers::SegmentationTokenizerRenderer,
+          :unicode_regex_tokenizer         => TwitterCldr::Js::Renderers::Tokenizers::UnicodeRegexTokenizerRenderer,
+          :range                           => TwitterCldr::Js::Renderers::Utils::RangeRenderer,
+          :range_set                       => TwitterCldr::Js::Renderers::Utils::RangeSetRenderer,
+          :code_points                     => TwitterCldr::Js::Renderers::Utils::CodePointsRenderer
         }
       end
     end

data/lib/twitter_cldr/js/mustache/calendars/datetime.coffee

@@ -42,10 +42,7 @@ class TwitterCldr.DateTimeFormatter
         when "pattern"
           this.result_for_token(token, obj)
         else
-          if token.value.length > 0 && token.value[0] == "'" && token.value[token.value.length - 1] == "'"
-            token.value.substring(1, token.value.length - 1)
-          else
-            token.value
+          token.value.replace(/'([^']+)'/g, '$1')
 
     tokens = this.get_tokens(obj, options)
     (format_token(token) for token in tokens).join("")

data/lib/twitter_cldr/js/mustache/numbers/numbers.coffee

@@ -22,8 +22,14 @@ class TwitterCldr.NumberFormatter
     tokens = this.get_tokens(number, opts)
 
     if tokens?
+      if tokens not instanceof Array
+        tokens_sample = tokens[Object.keys(tokens)[0]]
+        truncated_number = @truncate_number(number, tokens_sample[1].length)
+        truncated_number = Math.floor(truncated_number) if opts.precision == 0
+        tokens = tokens[TwitterCldr.PluralRules.rule_for(truncated_number)]
+
       [prefix, suffix, integer_format, fraction_format] = this.partition_tokens(tokens)
-      number = this.truncate_number(number, integer_format)
+      number = this.truncate_number(number, integer_format.format.length)
       [intg, fraction] = this.parse_number(number, opts)
       result = integer_format.apply(parseFloat(intg), opts)
       result += fraction_format.apply(fraction, opts) if fraction
@@ -33,7 +39,7 @@ class TwitterCldr.NumberFormatter
       # there's no specific formatting pattern for this number in current locale
       number.toString()
 
-  truncate_number: (number, integer_format) ->
+  truncate_number: (number, decimal_digits) ->
     number  # noop for base class
 
   partition_tokens: (tokens) ->
@@ -149,9 +155,9 @@ class TwitterCldr.AbbreviatedNumberFormatter extends TwitterCldr.NumberFormatter
     tokens = tokens[format] if format?
     tokens
 
-  truncate_number: (number, integer_format) ->
+  truncate_number: (number, decimal_digits) ->
     if @NUMBER_MIN <= number and number < @NUMBER_MAX
-      factor = Math.max(0, Math.floor(number).toString().length - integer_format.format.length)
+      factor = Math.max(0, Math.floor(number).toString().length - decimal_digits)
       number / Math.pow(10, factor)
     else
       number

data/lib/twitter_cldr/js/mustache/parsers/parser.coffee

@@ -0,0 +1,32 @@
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+class TwitterCldr.Parser
+
+  constructor : (@tokens) ->
+    @token_index = 0
+
+  parse : (tokens, options = {}) ->
+    @tokens = tokens
+    @reset()
+    @do_parse(options)
+
+  reset : ->
+    @token_index = 0
+
+  next_token : (type) ->
+    throw "Unexpected token" unless @current_token().type == type
+
+    @token_index += 1
+
+    while @current_token()? and @is_empty(@current_token())
+      @token_index += 1
+
+    @current_token()
+
+  is_empty : (token) ->
+    token.type == "plaintext" and token.value == ""
+
+  current_token : ->
+    @tokens[@token_index]
+

data/lib/twitter_cldr/js/mustache/parsers/segmentation_parser.coffee

@@ -0,0 +1,89 @@
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+class TwitterCldr.SegmentationParser extends TwitterCldr.Parser
+  constructor : ->
+    @begin_token ||= new TwitterCldr.Token({type : "special_char", value: "^"})
+    @regex_parser ||= new TwitterCldr.UnicodeRegexParser
+
+  class @.RuleMatchData
+    constructor : (@text, @boundary_offset) ->
+
+  class @.Rule
+    constructor : ->
+
+  class @.BreakRule extends @.Rule
+    constructor : (@left, @right) ->
+      @boundary_symbol = "break"
+      super
+
+    match : (str) ->
+      left_match = @left.match(str)
+
+      if @left? and left_match?
+        match_pos = str.indexOf(left_match[0]) + left_match[0].length
+
+        if @right?
+          right_match = @right.match(str.slice(match_pos))
+
+          if right_match?
+            return new TwitterCldr.SegmentationParser.RuleMatchData((left_match[0] + right_match[0]), match_pos)
+
+        else
+          return new TwitterCldr.SegmentationParser.RuleMatchData(str, str.length)
+      return null
+
+  class @.NoBreakRule extends @.Rule
+    constructor : (@regex) ->
+      @boundary_symbol = "no_break"
+      super
+
+    match : (str) ->
+      match = @regex.match(str)
+      if match?
+        new TwitterCldr.SegmentationParser.RuleMatchData(match[0], str.indexOf(match[0]) + match[0].length)
+      else
+        null
+
+  do_parse: (options = {}) ->
+    regex_token_lists = []
+    current_regex_tokens = []
+    boundary_symbol = null
+
+    while @current_token()?
+      switch @current_token().type
+        when "break", "no_break"
+          boundary_symbol = @current_token().type
+          regex_token_lists.push(current_regex_tokens)
+          current_regex_tokens = []
+        else
+          current_regex_tokens.push(@current_token())
+
+      @next_token(@current_token().type)
+
+    regex_token_lists.push(current_regex_tokens)
+
+    result = null
+
+    switch boundary_symbol
+      when "break"
+        result = new TwitterCldr.SegmentationParser.BreakRule(
+          @parse_regex(@add_anchors(regex_token_lists[0]), options),
+          @parse_regex(@add_anchors(regex_token_lists[1]), options)
+        )
+      when "no_break"
+        result = new TwitterCldr.SegmentationParser.NoBreakRule(
+          @parse_regex(
+            @add_anchors(
+              [].concat(regex_token_lists...)
+            ), options
+          )
+        )
+
+    result
+
+  add_anchors : (token_list) ->
+    [@begin_token].concat(token_list)
+
+  parse_regex : (tokens, options = {}) ->
+    if tokens? and tokens.length != 0 then new TwitterCldr.UnicodeRegex(@regex_parser.parse(tokens, options)) else null

data/lib/twitter_cldr/js/mustache/parsers/symbol_table.coffee

@@ -0,0 +1,14 @@
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+class TwitterCldr.SymbolTable
+  # This is really just a thin layer on top of Hash.
+  # Nice to have it abstracted in case we have to add custom behavior.
+
+  constructor : (@symbols = {}) ->
+
+  fetch : (symbol) ->
+    @symbols[symbol]
+
+  add : (symbol, value) ->
+    @symbols[symbol] = value

data/lib/twitter_cldr/js/mustache/parsers/unicode_regex/character_class.coffee

@@ -0,0 +1,51 @@
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+class TwitterCldr.CharacterClass extends TwitterCldr.Component
+  constructor : (@root) ->
+    @type = "character_class"
+    @grouping_pairs = TwitterCldr.CharacterClass.grouping_pairs
+    super
+
+  @grouping_pairs = {
+    "close_bracket" : "open_bracket"
+  }
+
+  @opening_types : ->
+    (value for key, value of @grouping_pairs)
+
+  @closing_types : ->
+    (key for key, value of @grouping_pairs)
+
+  @opening_type_for : (type) ->
+    if @grouping_pairs[type]? then @grouping_pairs[type] else null
+
+  to_regexp_str : ->
+    @set_to_regex(@to_set())
+
+  to_set : ->
+    @evaluate(@root)
+
+  evaluate : (node) ->
+    if node instanceof TwitterCldr.CharacterClass.UnaryOperator or node instanceof TwitterCldr.CharacterClass.BinaryOperator
+      switch node.operator
+        when "negate"
+          TwitterCldr.UnicodeRegex.get_valid_regexp_chars().subtract(@evaluate(node.child))
+        when "union", "pipe"
+          @evaluate(node.left).union(@evaluate(node.right))
+        when "dash"
+          @evaluate(node.left).difference(@evaluate(node.right))
+        when "ampersand"
+          @evaluate(node.left).intersection(@evaluate(node.right))
+
+    else
+      if node?
+        node.to_set()
+      else
+        new TwitterCldr.RangeSet([])
+
+  class @.BinaryOperator
+    constructor : (@operator, @left, @right) ->
+
+  class @.UnaryOperator
+    constructor : (@operator, @child) ->

data/lib/twitter_cldr/js/mustache/parsers/unicode_regex/character_range.coffee

@@ -0,0 +1,19 @@
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+class TwitterCldr.CharacterRange extends TwitterCldr.Component
+
+  constructor : (@initial, @final) ->
+    # Unfortunately, due to the ambiguity of having both character
+    # ranges and set operations in the same syntax (which both use
+    # the "-" operator and square brackets), we have to treat
+    # CharacterRange as both a token and an operand. This type method
+    # helps it behave like a token.
+    @type = "character_range"
+    super
+
+  to_set : ->
+    new TwitterCldr.RangeSet([new TwitterCldr.Range(
+        @initial.to_set().to_full_array()[0],
+        @final.to_set().to_full_array()[0])
+    ])

data/lib/twitter_cldr/js/mustache/parsers/unicode_regex/character_set.coffee

@@ -0,0 +1,36 @@
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+class TwitterCldr.CharacterSet extends TwitterCldr.Component
+  constructor : (text) ->
+    name_parts = text.split("=")
+    if name_parts.length == 2
+      @property = name_parts[0].toLowerCase()
+      @property_value = name_parts[1]
+    else
+      @property_value = text
+      @property = null
+
+  to_regexp_str : ->
+    @set_to_regex(@to_set())
+
+  to_set : ->
+    @codepoints().subtract(
+      TwitterCldr.UnicodeRegex.get_unsupported_chars()
+    ).subtract(
+      TwitterCldr.UnicodeRegex.get_invalid_regexp_chars()
+    )
+
+  codepoints : ->
+    if @property?
+      method = "code_points_for_" + @property
+      ranges = TwitterCldr.CodePoint.code_points_for_property(@property, @property_value)
+
+      if ranges?
+        new TwitterCldr.RangeSet(ranges)
+      else
+        throw "Couldn't find property " + @property + " containing property value " + @property_value
+    else
+      new TwitterCldr.RangeSet(
+        TwitterCldr.CodePoint.code_points_for_property_value(@property_value)
+      )

data/lib/twitter_cldr/js/mustache/parsers/unicode_regex/component.coffee

@@ -0,0 +1,48 @@
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+class TwitterCldr.Component
+  to_utf8 : (codepoints) ->
+    unless codepoints instanceof Array
+      codepoints = [codepoints]
+
+    (@to_hex(cp) for cp in codepoints)
+
+  to_hex : (codepoint) ->
+    if (codepoint >= 0 and codepoint <= 0xD7FF or codepoint >= 0xE000 and codepoint <= 0xFFFF)
+      return @to_escaped_hex(codepoint)
+    else if (codepoint >= 0x10000 and codepoint <= 0x10FFFF)
+      codepoint -= 0x10000
+      first = ((0xffc00 & codepoint) >> 10) + 0xD800
+      second = (0x3ff & codepoint) + 0xDC00
+      return @to_escaped_hex(first) + '+' + @to_escaped_hex(second)
+
+  to_escaped_hex : (codepoint) ->
+    s = codepoint.toString(16)
+    s = "0000".slice(0, 4 - s.length) + s
+    return "\\u" + s;
+
+  range_to_regex : (range) ->
+    if range.first instanceof Array
+      @array_to_regex(range)
+    else
+      "[" + @to_utf8(range.first) + "-" + @to_utf8(range.last) + "]"
+
+  array_to_regex : (arr) ->
+    ("(?:" + @to_utf8(c) + ")" for c in arr).join("")
+
+  set_to_regex : (set) ->
+    strs = ((@._set_element_to_regex(element)
+    ) for element in TwitterCldr.Utilities.remove_duplicates(
+      set.to_array(true))
+    )
+
+    ("(?:" + strs.join("|") + ")")
+
+  _set_element_to_regex : (element) ->
+    if element instanceof TwitterCldr.Range
+      @range_to_regex(element)
+    else if element instanceof Array
+      @array_to_regex(element)
+    else
+      @to_utf8(element)

data/lib/twitter_cldr/js/mustache/parsers/unicode_regex/literal.coffee

@@ -0,0 +1,44 @@
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+class TwitterCldr.Literal extends TwitterCldr.Component
+
+  constructor : (@text) ->
+    @special_characters = {
+      s : [32]  # space
+      t : [9]   # tab
+      r : [13]  # carriage return
+      n : [10]  # newline
+      f : [12]  # form feed
+      d : (char for char in [48..57]) # digits 0-9
+      w : (char for char in [97..122].concat([65..90]).concat([48..57]).concat([95])) # lowercase, uppercase, numbers
+    }
+
+
+    super
+
+  ordinalize : (char) ->
+    TwitterCldr.Utilities.char_code_at(char, 0)
+
+  to_regexp_str : ->
+    @text
+
+  to_set : ->
+    if @text.match(/^\\/)
+      special_char = @text.slice(1)
+
+      if @special_characters[special_char.toLowerCase()]?
+        @set_for_special_char(special_char)
+
+      else
+        TwitterCldr.RangeSet.from_array([@ordinalize(special_char)])
+
+    else
+      TwitterCldr.RangeSet.from_array([@ordinalize(@text)])
+
+  set_for_special_char : (char) ->
+    chars = TwitterCldr.RangeSet.from_array(@special_characters[char.toLowerCase()])
+    if char.toUpperCase() == char
+      TwitterCldr.UnicodeRegex.get_valid_regexp_chars().subtract(chars)
+    else
+      chars