twitter_cldr_js 2.3.2 → 2.4.0

data/lib/twitter_cldr/js/mustache/parsers/unicode_regex/unicode_string.coffee

@@ -0,0 +1,23 @@
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+class TwitterCldr.UnicodeString extends TwitterCldr.Component
+  constructor : (@codepoints) ->
+    super
+
+  to_set : ->
+    # If the number of codepoints is greater than 1, treat them as a
+    # group (eg. multichar string). This is definitely a hack in that
+    # it means there has to be special logic in RangeSet that deals
+    # with data types that aren't true integer ranges. I can't think
+    # of any other way to support multichar strings :(
+
+    if @codepoints.length > 1
+      new TwitterCldr.RangeSet([new TwitterCldr.Range @codepoints, @codepoints])
+    else
+      new TwitterCldr.RangeSet([new TwitterCldr.Range @codepoints[0], @codepoints[0]])
+
+  to_regexp_str : ->
+    cps = (if @codepoints instanceof Array then @codepoints else [@codepoints])
+    @array_to_regex(cps)
+

data/lib/twitter_cldr/js/mustache/parsers/unicode_regex_parser.coffee

@@ -0,0 +1,189 @@
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+class TwitterCldr.UnicodeRegexParser extends TwitterCldr.Parser
+
+  constructor : ->
+    # Types that are allowed to be used in character ranges.
+    @character_class_token_types = [
+      "variable", "character_set", "negated_character_set", "unicode_char",
+      "multichar_string", "string", "escaped_character", "character_range"
+    ]
+
+    @negated_token_types = [
+      "negated_character_set"
+    ]
+
+    @binary_operators = [
+      "pipe", "ampersand", "dash", "union"
+    ]
+
+    @unary_operators = [
+      "negate"
+    ]
+
+  parse : (tokens, options = {}) ->
+    super(@preprocess(@substitute_variables(tokens, options.symbol_table)), options)
+
+  make_token : (type, value) ->
+    new TwitterCldr.Token ({"type": type, "value" : value})
+
+  # Identifies regex ranges and makes implicit operators explicit
+  preprocess : (tokens) ->
+    result = []
+    i = 0
+
+    while i < tokens.length
+      # Character class entities side-by-side are treated as unions. So
+      # are side-by-side character classes. Add a special placeholder token
+      # to help out the expression parser.
+      add_union = (@is_valid_character_class_token(result[result.length-1]) and tokens[i].type != "close_bracket") ||
+            (result[result.length-1]? and result[result.length-1].type == "close_bracket" and tokens[i].type == "open_bracket")
+      result.push(@make_token("union")) if add_union
+
+      is_range = @is_valid_character_class_token(tokens[i]) and
+            @is_valid_character_class_token(tokens[i + 2]) and
+            tokens[i + 1].type == "dash"
+      if is_range
+        initial = @[tokens[i].type](tokens[i])
+        final = @[tokens[i+2].type](tokens[i+2])
+        result.push(@make_character_range(initial, final))
+        i += 3
+      else
+        if @is_negated_token(tokens[i])
+          result = result.concat [
+            @make_token("open_bracket")
+            @make_token("negate")
+            tokens[i]
+            @make_token("close_bracket")
+          ]
+        else
+          result.push(tokens[i])
+
+        i += 1
+
+    result
+
+  substitute_variables : (tokens, symbol_table) ->
+    return tokens unless symbol_table?
+
+    result = []
+    for i in [0...tokens.length] by 1
+      token = tokens[i]
+      if token.type == "variable" and (sub = symbol_table.fetch(token.value))?
+        # variables can themselves contain references to other variables
+        # note: this could be cached somehow
+        result = result.concat(@substitute_variables(sub, symbol_table))
+      else
+        result.push token
+
+    result
+
+  make_character_range : (initial, final) ->
+    new TwitterCldr.CharacterRange(initial, final)
+
+  is_negated_token : (token) ->
+    token? and token.type in @negated_token_types
+
+  is_valid_character_class_token : (token) ->
+    token? and token.type in @character_class_token_types
+
+  is_unary_operator : (token) ->
+    token? and token.type in @unary_operators
+
+  is_binary_operator : (token) ->
+    token? and token.type in @binary_operators
+
+  do_parse : (options) ->
+    elements = []
+    while @current_token()
+      switch @current_token().type
+        when "open_bracket"
+          elements.push(@character_class())
+        when "union"
+          @next_token("union")
+        else
+          elements.push (@[@current_token().type](@current_token()))
+          @next_token(@current_token().type)
+    elements
+
+  character_set : (token) ->
+    new TwitterCldr.CharacterSet(token.value.replace(/^\\p/g, "").replace(/[\{\}\[\]:]/g, ""))
+
+  negated_character_set : (token) ->
+    new TwitterCldr.CharacterSet(token.value.replace(/^\\[pP]/g, "").replace(/[\{\}\[\]:^]/g, ""))
+
+  unicode_char : (token) ->
+    new TwitterCldr.UnicodeString([parseInt(token.value.replace(/^\\u/g, "").replace(/[\{\}]/g, ""), 16)])
+
+  string : (token) ->
+    new TwitterCldr.UnicodeString(TwitterCldr.Utilities.unpack_string(token.value))
+
+  multichar_string : (token) ->
+    new TwitterCldr.UnicodeString(TwitterCldr.Utilities.unpack_string(token.value.replace(/[\{\}]/g, "")))
+
+  escaped_character : (token) ->
+    new TwitterCldr.Literal(token.value)
+
+  special_char : (token) ->
+    new TwitterCldr.Literal(token.value)
+
+  negate : (token) ->
+    @special_char(token)
+
+  pipe : (token) ->
+    @special_char(token)
+
+  ampersand : (token) ->
+    @special_char(token)
+
+
+  # current_token is already a CharacterRange object
+  character_range : (token) ->
+    token
+
+  character_class : ->
+    operator_stack = []
+    operand_stack = []
+    open_count = 0
+
+    while true
+      if @current_token().type in TwitterCldr.CharacterClass.closing_types()
+        last_operator = @peek(operator_stack)
+        open_count -= 1
+        while last_operator.type isnt TwitterCldr.CharacterClass.opening_type_for(@current_token().type)
+          operator = operator_stack.pop()
+          node = if @is_unary_operator(operator)
+            @unary_operator_node(operator.type, operand_stack.pop())
+          else
+            @binary_operator_node(operator.type, operand_stack.pop(), operand_stack.pop())
+
+          operand_stack.push(node)
+          last_operator = @peek(operator_stack)
+
+        operator_stack.pop()
+
+      else if @current_token().type in TwitterCldr.CharacterClass.opening_types()
+        open_count += 1
+        operator_stack.push(@current_token())
+
+      else if @current_token().type in @unary_operators.concat(@binary_operators)
+        operator_stack.push(@current_token())
+
+      else
+        operand_stack.push(@[@current_token().type](@current_token()))
+
+      @next_token(@current_token().type)
+
+      break if operator_stack.length is 0 and open_count is 0
+
+    new TwitterCldr.CharacterClass(operand_stack.pop())
+
+  peek : (array) ->
+    array[array.length-1]
+
+  binary_operator_node : (operator, right, left) ->
+    new TwitterCldr.CharacterClass.BinaryOperator(operator, left, right)
+
+  unary_operator_node : (operator, child) ->
+    new TwitterCldr.CharacterClass.UnaryOperator(operator, child)

data/lib/twitter_cldr/js/mustache/plurals/rules.coffee

@@ -3,12 +3,14 @@
 
 class TwitterCldr.PluralRules
   @rules = `{{{rules}}}`
+  @runtime = `{{{runtime}}}`
+  @names = {{{names}}}
 
-  @all: ->
-    return @rules.keys
+  @all: (type = 'cardinal') ->
+    return @names[type]
 
-  @rule_for: (number) ->
+  @rule_for: (number, type = 'cardinal') ->
     try
-      return @rules.rule(number)
+      return @rules[type](number.toString(), @runtime)
     catch error
-      return "other"
+      return "other"

data/lib/twitter_cldr/js/mustache/shared/break_iterator.coffee

@@ -0,0 +1,148 @@
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+class TwitterCldr.BreakIterator
+  constructor : (locale = TwitterCldr.locale, options = {}) ->
+    @locale = locale
+    @use_uli_exceptions = (if options["use_uli_exceptions"]? then options["use_uli_exceptions"] else true)
+    @exceptions_cache = {}
+    @segmentation_tokenizer = new TwitterCldr.SegmentationTokenizer()
+    @segmentation_parser = new TwitterCldr.SegmentationParser()
+    @tailoring_resource_data = `{{{tailoring_resource_data}}}`
+    @exceptions_resource_data = `{{{exceptions_resource_data}}}`
+    @root_resource = `{{{root_resource_data}}}`
+
+  each_sentence : (str, block) ->
+    @each_boundary(str, "sentence", block)
+
+  each_word : (str, block) ->
+    throw "Word segmentation is not currently supported."
+
+  each_line : (str, block) ->
+    throw "Line segmentation is not currently supported."
+
+  boundary_name_for: (str) ->
+    str.replace(/(?:^|\_)([A-Za-z])/, (match) ->
+      match.toUpperCase()
+      ) + "Break"
+
+  each_boundary : (str, boundary_type, block) ->
+    rules = @compile_rules_for(@locale, boundary_type)
+    match = null
+    last_offset = 0
+    current_position = 0
+    search_str = str
+    result = []
+
+    while(search_str.length isnt 0)
+      rule = null
+      for r in rules
+        match = r.match(search_str)
+        if match?
+          rule = r
+          break
+      if rule.boundary_symbol is "break"
+        break_offset = current_position + match.boundary_offset
+        result.push(str.slice(last_offset, break_offset))
+        if block?
+          block(result[result.length-1])
+
+        last_offset = break_offset
+
+      search_str = search_str.slice(match.boundary_offset)
+      current_position += match.boundary_offset
+
+    if last_offset < str.length - 1
+      result.push(str.slice(last_offset))
+      if block?
+        block(str.slice(last_offset))
+
+
+    result
+
+  compile_exception_rule_for : (locale, boundary_type, boundary_name) ->
+    if boundary_type is "sentence"
+      cache_key = TwitterCldr.Utilities.compute_cache_key([locale, boundary_type])
+
+      result = null
+      exceptions = @exceptions_for(locale, boundary_name)
+      regex_contents = (TwitterCldr.Utilities.regex_escape(exception) for exception in exceptions).join("|")
+      @exceptions_cache[cache_key] ||=  @segmentation_parser.parse (
+        @segmentation_tokenizer.tokenize("(?:"+regex_contents+") \u00D7")
+      )
+
+  # Grabs rules from segment_root, applies custom tailorings (our own, NOT from CLDR),
+  # and optionally integrates ULI exceptions.
+  compile_rules_for : (locale, boundary_type) ->
+    boundary_name = @boundary_name_for(boundary_type)
+    boundary_data = @resource_for(boundary_name)
+    symbol_table = @symbol_table_for(boundary_data)
+    root_rules = @rules_for(boundary_data, symbol_table)
+    tailoring_boundary_data = @tailoring_resource_for(locale, boundary_name)
+    tailoring_rules = @rules_for(tailoring_boundary_data, symbol_table)
+    rules = @merge_rules(root_rules, tailoring_rules)
+
+    if @use_uli_exceptions is true
+      exception_rule = @compile_exception_rule_for(locale, boundary_type, boundary_name)
+      rules.unshift(exception_rule)
+
+    rules
+
+  # replaces ruleset1's rules with rules with the same id from ruleset2
+  merge_rules : (ruleset1, ruleset2) ->
+    result = []
+    TwitterCldr.Utilities.arraycopy ruleset1, 0, result, 0, ruleset1.length
+
+    for i in [0...ruleset2.length] by 1
+      for j in [0...result.length] by 1
+        if ruleset2[i].id == result[j].id
+          result[j] = ruleset2[i]
+
+    result
+
+  symbol_table_for : (boundary_data) ->
+    table = new TwitterCldr.SymbolTable()
+
+    for i in [0...boundary_data.variables.length] by 1
+      variable = boundary_data.variables[i]
+      id = variable.id.toString()
+      tokens = @segmentation_tokenizer.tokenize(variable.value)
+      # note: variables can be redefined (add replaces if key already exists)
+      table.add(id, @resolve_symbols(tokens, table))
+
+    table
+
+  resolve_symbols : (tokens, symbol_table) ->
+    result = []
+
+    for i in [0...tokens.length]
+      token = tokens[i]
+      if token.type == "variable"
+        result = result.concat(symbol_table.fetch(token.value))
+      else
+        result.push(token)
+
+    result
+
+  rules_for : (boundary_data, symbol_table) ->
+    results = []
+    for rule in boundary_data.rules
+      r = @segmentation_parser.parse(
+        @segmentation_tokenizer.tokenize(rule.value), {"symbol_table" : symbol_table}
+      )
+      r.string = rule.value
+      r.id = rule.id
+      results.push(r)
+
+    results
+
+
+  resource_for : (boundary_name) ->
+    @root_resource["segments"][boundary_name]
+
+  tailoring_resource_for : (locale, boundary_name) ->
+    @tailoring_resource_data[locale][locale]["segments"][boundary_name]
+
+  exceptions_for : (locale, boundary_name) ->
+    result = @exceptions_resource_data[locale][locale]["exceptions"]
+    if result? then result else []

data/lib/twitter_cldr/js/mustache/shared/code_point.coffee

@@ -0,0 +1,121 @@
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+class TwitterCldr.CodePoint
+  @code_point_fields = [
+    "code_point"
+    "name"
+    "category"
+    "combining_class"
+    "bidi_class"
+    "decomposition"
+    "digit_value"
+    "non_decimal_digit_value"
+    "numeric_value"
+    "bidi_mirrored"
+    "unicode1_name"
+    "iso_comment"
+    "simple_uppercase_map"
+    "simple_lowercase_map"
+    "simple_titlecase_map"
+  ]
+
+  decomposition_data_index = 5
+  decomposition_regex = /^(?:<(.+)>\s+)?(.+)?$/
+  @indices = ["category", "bidi_class", "bidi_mirrored"]
+  @properties = ["sentence_break", "line_break", "word_break"]
+
+  constructor : (@fields) ->
+    for i in [0...TwitterCldr.CodePoint.code_point_fields.length] by 1
+      field = TwitterCldr.CodePoint.code_point_fields[i]
+      unless field is "decomposition"
+        @[field] = @fields[i]
+
+
+  decomposition : ->
+    decomp = @fields[decomposition_data_index]
+    match = decomp.match(decomposition_regex)
+    if match?
+      if match[2]?
+        return (parseInt(s, 16) for s in match[2].match(/\S+/g))
+      else
+        return null
+    else
+      throw "decomposition " + decomp + " has invalid format"
+
+  compatibility_decomposition_tag : ->
+    decomp = @fields[decomposition_data_index]
+    if (match = decomp.match(decomposition_regex))
+      if match[1]? then return match[1] else return null
+    else
+      throw "decomposition " + decomp + " has invalid format"
+
+  is_compatibility_decomposition : ->
+    return @compatibility_decomposition_tag()?
+
+  @code_points_for_property : (property_name, value)  ->
+    property_data = @get_property_data(property_name)
+    if property_data?
+      property_data[value]
+    else
+          throw "Couldn't find property " + property_name
+
+  # Search for code points wherein at least one property value contains prop_value.
+  # For example, if prop_value is set to "Zs", this method will return all code
+  # points that are considered spaces. If prop value is simply "Z", this method
+  # will return all code points who have a property value that contains "Z", i.e.
+  # spaces as well as line separators ("Zl") and paragraph separators ("Zp").
+  @code_points_for_property_value : (prop_value) ->
+    if @index_key_cache[prop_value]?
+      return @index_key_cache[prop_value]
+
+    result = []
+    for index_key, index_names of @index_keys
+      if index_key.indexOf(prop_value) > -1
+        for index_name in index_names
+          result = result.concat(@get_index(index_name)[index_key])
+
+    @index_key_cache[prop_value] = result
+
+  @index_key_cache = {}
+
+  @index_keys = `{{{index_keys}}}`
+
+  @index_data = `{{{index_data}}}`
+
+  @get_index : (index_name) ->
+    return @index_cache[index_name] if @index_cache[index_name]?
+    index_data = @index_data[index_name]
+    index_data_formatted = {}
+    for k, v of index_data
+      index_data_formatted[k] = []
+      for range in index_data[k]
+        index_data_formatted[k].push(new TwitterCldr.Range(range[0], range[1]))
+
+    @index_cache[index_name] = index_data_formatted
+
+  @property_data = `{{{property_data}}}`
+
+  @get_property_data : (property_name) ->
+    return @property_data_cache[property_name] if @property_data_cache[property_name]?
+    property_data = @property_data[property_name]
+    property_data_formatted = {}
+    for k, v of property_data
+      property_data_formatted[k] = []
+      for range in property_data[k]
+        property_data_formatted[k].push(new TwitterCldr.Range(range[0], range[1]))
+
+    @property_data_cache[property_name] = property_data_formatted
+
+  @index_cache = {}
+
+  @property_data_cache = {}
+
+  @get_block_name : (code_point) ->
+    if @block_cache[code_point]?
+      return @block_cache[code_point]
+    for k, range of @blocks
+      range = new TwitterCldr.Range(range[0], range[1])
+      if range.includes(code_point)
+        return @block_cache[code_point] = k
+    return null