twitter_cldr_js 2.3.2 → 2.4.0

data/lib/twitter_cldr/js/mustache/shared/unicode_regex.coffee

@@ -0,0 +1,41 @@
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+class TwitterCldr.UnicodeRegex
+  constructor : (@elements, @modifiers = "") ->
+
+  @compile : (str, modifiers = "", symbol_table = null) ->
+    new TwitterCldr.UnicodeRegex(@get_parser().parse(
+        @get_tokenizer().tokenize(str), {"symbol_table" : symbol_table}
+      ), modifiers)
+
+  @get_all_unicode : ->
+    @all_unicode ||= new TwitterCldr.RangeSet([new TwitterCldr.Range(0, 0xFFFF)])
+
+  # A few <control> characters (i.e. 2..7) and public/private surrogates (i.e. 55296..57343).
+  # These don't play nicely with Ruby's regular expression engine, and I think we
+  # can safely disregard them.
+  @get_invalid_regexp_chars : ->
+    @invalid_regexp_chars ||= new TwitterCldr.RangeSet([(new TwitterCldr.Range(2, 7)),
+              (new TwitterCldr.Range(55296, 57343))])
+
+  @get_valid_regexp_chars : ->
+    @valid_regexp_chars ||= @get_all_unicode().subtract(@get_invalid_regexp_chars())
+
+  @get_unsupported_chars : ->
+    @unsupported_chars ||= new TwitterCldr.RangeSet([new TwitterCldr.Range(0x10000, 0x10FFFF)])
+
+  @get_tokenizer : ->
+    @tokenizer = new TwitterCldr.UnicodeRegexTokenizer()
+
+  @get_parser : ->
+    @parser = new TwitterCldr.UnicodeRegexParser()
+
+  to_regexp_str : ->
+    @regexp_str ||= ((element.to_regexp_str()) for element in @elements).join("")
+
+  to_regexp : ->
+    new RegExp(@to_regexp_str(), @modifiers)
+
+  match : (str) ->
+    str.match(@to_regexp())

data/lib/twitter_cldr/js/mustache/tokenizers/composite_token.coffee

@@ -0,0 +1,11 @@
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+class TwitterCldr.CompositeToken
+  constructor : (@tokens) ->
+    @type = "composite"
+
+  to_string : ->
+    unless @tokens?
+      return null
+    (token.to_string() for token in @tokens).join("")

data/lib/twitter_cldr/js/mustache/tokenizers/segmentation_tokenizer.coffee

@@ -0,0 +1,24 @@
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+class TwitterCldr.SegmentationTokenizer
+  constructor : ->
+    recognizers = [
+      new TwitterCldr.TokenRecognizer("break", /\u00f7/, ((val) -> 
+        TwitterCldr.Utilities.trim_string(val))) # ÷ character
+      new TwitterCldr.TokenRecognizer("no_break", /\u00d7/, ((val) -> 
+        TwitterCldr.Utilities.trim_string(val))) # × character
+    ]
+    ur_tokenizer = new TwitterCldr.UnicodeRegexTokenizer
+    ur_tokenizer.insert_before("string", recognizers)
+    @tokenizer = ur_tokenizer
+    
+
+  tokenize : (pattern) ->
+    result = []
+    tokens = @tokenizer.tokenize pattern
+    for token in tokens
+      if token.value.replace(/^\s+|\s+$/g, "").length isnt  0
+        result.push token
+
+    result

data/lib/twitter_cldr/js/mustache/tokenizers/token.coffee

@@ -0,0 +1,14 @@
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+class TwitterCldr.Token
+
+  constructor : (options = {}) ->
+    for k, v of options
+      @[k] = v
+  
+  to_hash : ->
+    { "value" : @value, "type" : @type }
+
+  to_string : ->
+    @value

data/lib/twitter_cldr/js/mustache/tokenizers/tokenizer.coffee

@@ -0,0 +1,83 @@
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+class TwitterCldr.TokenRecognizer
+  constructor : (@token_type, @regex, @cleaner, @content = null) ->
+
+  recognizes : (text) ->
+    @regex.test text
+
+  clean : (val) ->
+    if @cleaner? then @cleaner(val) else val
+
+class TwitterCldr.Tokenizer
+  constructor : (@recognizers, @custom_splitter = null, @remove_empty_entries = true) ->
+    @splitter = (@custom_splitter || new RegExp("(" + ((recognizer.regex.source
+    ) for recognizer in recognizers
+    ).join("|") + ")"))
+
+  @union : (tokenizers, block) ->
+    recognizers = []
+    for tokenizer in tokenizers
+      recog_ret = []
+      for recognizer in tokenizer.recognizers
+        if block? and block(recognizer) || !block?
+          recog_ret.push recognizer
+      recognizer.concat(recog_ret)
+
+    flag = true
+    for tokenizer in tokenizers
+      if (custom_splitter? and !(@custom_splitter(tokenizer)))
+        flag = false
+    splitter = null
+    if flag
+      splitter = new Regexp(
+        ((tokenizer.custom_splitter.source
+        ) for tokenizer in tokenizers).join("|")
+      )
+    new TwitterCldr.Tokenizer(recognizers, splitter)
+
+  recognizer_at : (token_type) ->
+    recognizer for recognizer in @recognizers when recognizer.token_type is token_type
+
+  insert_before : (token_type, new_recognizers) ->
+
+    idx = 0
+    for i in [0...@recognizers.length]
+      recognizer = @recognizers[i]
+      if recognizer.token_type is token_type
+        idx = i
+    for recognizer in new_recognizers
+      @recognizers.splice(idx, 0, recognizer)
+      idx += 1
+    @clear_splitter()
+    null
+
+  tokenize : (text) ->
+    pieces = text.match(new RegExp(@get_splitter().source, "g"))
+    result = []
+    for piece in pieces
+      recognizer = null
+      for r in @recognizers
+        if r.recognizes(piece)
+          recognizer = r
+          break
+
+      if recognizer.token_type is "composite"
+        content = piece.match(recognizer.content)[0]
+        result.push(new TwitterCldr.CompositeToken(@tokenize(content)))
+
+      else
+        cleaned_text = recognizer.clean(piece)
+        if ((@remove_empty_entries and cleaned_text.length > 0) or !@remove_empty_entries)
+          result.push(new TwitterCldr.Token({"value" : cleaned_text, "type" : recognizer.token_type}))
+
+    result
+
+  clear_splitter : ->
+    @splitter = null
+
+  get_splitter : ->
+    @splitter = (@custom_splitter || new RegExp("(" + ((recognizer.regex.source
+    ) for recognizer in @recognizers
+    ).join("|") + ")"))

data/lib/twitter_cldr/js/mustache/tokenizers/unicode_regex/unicode_regex_tokenizer.coffee

@@ -0,0 +1,39 @@
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+class TwitterCldr.UnicodeRegexTokenizer
+  constructor : ->
+    recognizers = [
+      # The variable name can contain letters and digits, but must start with a letter.
+      new TwitterCldr.TokenRecognizer("variable", new RegExp(/\$\w[\w\d]*/)),
+      new TwitterCldr.TokenRecognizer("character_set", new RegExp(/\[:\w+:\]|\\p\{[\w=]+\}/)),  # [:Lu:] or \p{Lu} or \p{Sentence_Break=CF}
+      new TwitterCldr.TokenRecognizer("negated_character_set", new RegExp(/\[:\^\w+:\]|\\P\{[\w=]+\}/)),  #[:^Lu:] or \P{Lu}
+      new TwitterCldr.TokenRecognizer("unicode_char", new RegExp(/\\u\{?[a-fA-F0-9]{1,6}\}?/)),
+      new TwitterCldr.TokenRecognizer("multichar_string", new RegExp(/\{\w+\}/)),
+
+      new TwitterCldr.TokenRecognizer("escaped_character", new RegExp(/\\./)),
+      new TwitterCldr.TokenRecognizer("negate", new RegExp(/\^/)),
+      new TwitterCldr.TokenRecognizer("ampersand", new RegExp(/&/)),
+      new TwitterCldr.TokenRecognizer("pipe", new RegExp(/\|/)),
+      new TwitterCldr.TokenRecognizer("dash", new RegExp(/-/)),
+
+      # stuff that shouldn't be converted to codepoints
+      new TwitterCldr.TokenRecognizer("special_char", new RegExp(/\{\d,?\d?\}|[$?:{}()*+\.,\/\\]/)),
+
+      new TwitterCldr.TokenRecognizer("open_bracket", new RegExp(/\[/)),
+      new TwitterCldr.TokenRecognizer("close_bracket", new RegExp(/\]/)),
+
+      new TwitterCldr.TokenRecognizer("string", new RegExp(/[\s\S]/), ((val) ->
+          if val is " "
+            val
+          else
+            TwitterCldr.Utilities.trim_string(val)
+        ))
+    ]
+    @tokenizer = new TwitterCldr.Tokenizer(recognizers)
+
+  insert_before : (token_type, new_recognizers) ->
+    @tokenizer.insert_before(token_type, new_recognizers)
+
+  tokenize : (pattern) ->
+    @tokenizer.tokenize(pattern)

data/lib/twitter_cldr/js/mustache/utilities.coffee

@@ -85,3 +85,48 @@ class TwitterCldr.Utilities
 
   @is_odd: (num) ->
     num % 2 == 1
+
+  @remove_duplicates : (arr) ->
+    arr.reduce ((u, elem) ->
+      u.push elem  if u.indexOf(elem) < 0
+      u
+    ), []
+
+  # This function was adapted from the Mozilla JS reference:
+  # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions
+  @regex_escape : (s) ->
+    s.replace(/([.*+?^=!:${}()|\[\]\/\\])/g, "\\$1")
+
+  # This function was adapted from the Mozilla JS reference:
+  # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/Trim
+  @trim_string : (s) ->
+    s.replace(/^\s+|\s+$/g, '')
+
+  # This function was adapted from the CoffeeScript Cookbook Reference:
+  # http://coffeescriptcookbook.com/chapters/classes_and_objects/cloning
+  @clone : (obj) ->
+    if not obj? or typeof obj isnt 'object'
+      return obj
+
+    if obj instanceof Date
+      return new Date(obj.getTime()) 
+
+    if obj instanceof RegExp
+      flags = ''
+      flags += 'g' if obj.global?
+      flags += 'i' if obj.ignoreCase?
+      flags += 'm' if obj.multiline?
+      flags += 'y' if obj.sticky?
+      return new RegExp(obj.source, flags) 
+
+    newInstance = new obj.constructor()
+
+    for key of obj
+      newInstance[key] = @clone obj[key]
+
+    return newInstance
+
+  @compute_cache_key : (pieces) ->
+    if pieces? and pieces.length > 0
+      return pieces.join("|")
+    return null

data/lib/twitter_cldr/js/mustache/utils/code_points.coffee

@@ -0,0 +1,23 @@
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+class TwitterCldr.CodePoints
+
+  @to_char : (code_point) ->
+    TwitterCldr.Utilities.pack_array([code_point])
+
+  @from_char : (char) ->
+    TwitterCldr.Utilities.unpack_string(char[0])[0]
+
+  @from_chars : (chars) ->
+    (@from_char(char) for char in chars)
+
+  @to_chars : (code_points) ->
+    (@to_char(code_point) for code_point in code_points)
+
+  @from_string : (str) ->
+    TwitterCldr.Utilities.unpack_string(str)
+
+  @to_string : (code_points) ->
+    @to_chars(code_points).join("")
+

data/lib/twitter_cldr/js/mustache/utils/range.coffee

@@ -0,0 +1,16 @@
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+class TwitterCldr.Range
+  constructor : (@first, @last) ->
+    if @is_numeric()
+      @size = @last - @first + 1
+
+  to_array : ->
+    [@first..@last]
+
+  includes : (number) ->
+    number >= @first and number <= @last
+
+  is_numeric : ->
+    (!(@first instanceof Array) and !(@last instanceof Array))

data/lib/twitter_cldr/js/mustache/utils/range_set.coffee

@@ -0,0 +1,195 @@
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+class TwitterCldr.RangeSet
+  # An integer set, implemented under the hood with ranges. The idea is
+  # that it's more efficient to store sequential data in ranges rather
+  # than as single elements. By definition, RangeSets contain no duplicates.
+  constructor : (ranges) ->
+    @ranges = []
+    for range in ranges
+      if range instanceof TwitterCldr.Range
+        @ranges.push (new TwitterCldr.Range(range.first, range.last))
+      else
+        @ranges.push (new TwitterCldr.Range(range, range))
+    @flatten()
+
+  @from_array : (array, compress = false) ->
+    new TwitterCldr.RangeSet @rangify(array, compress)
+
+  # Turns an array of integers into ranges. The "compress" option indicates
+  # wether or not to turn isolated elements into zero-length ranges or leave
+  # them as single elements.
+  #
+  # For example:
+  # rangify([1, 2, 4], false) returns [1..2, 4..4]
+  # rangify([1, 2, 4], true) returns [1..2, 4]
+  @rangify : (list, compress = false) ->
+    last_item = null
+
+    sorted_list = list.sort( (a,b) ->
+      a - b
+    )
+    sub_lists = []
+    for item in sorted_list
+      if last_item?
+        diff = item - last_item
+
+        if diff > 0
+          if diff is 1
+            sub_lists[sub_lists.length-1].push(item)
+          else
+            sub_lists.push([item])
+          last_item = item
+      else
+        sub_lists.push([item])
+        last_item = item
+
+    ( (if compress && sub_list.length is 1
+        sub_list[0]
+      else
+        new TwitterCldr.Range(sub_list[0], sub_list[sub_list.length-1])
+      ) for sub_list in sub_lists)
+
+
+  to_array : (compress = false) ->
+    if compress
+      ( (if range.first is range.last
+          range.first
+        else
+          TwitterCldr.Utilities.clone(range)
+        ) for range in @ranges)
+    else
+      TwitterCldr.Utilities.clone(@ranges)
+
+  to_full_array : ->
+    result = []
+
+    for range in @ranges
+      result = result.concat(range.to_array())
+
+    result
+
+  includes : (obj) ->
+    if obj instanceof TwitterCldr.Range
+      for range in @ranges
+        return true if range.first <= obj.first && range.last >= obj.last
+    else
+      for range in @ranges
+          return true if range.includes(obj)
+    false
+
+  is_empty : ->
+    @ranges.length == 0
+
+  union : (range_set) ->
+    new TwitterCldr.RangeSet @ranges.concat(range_set.ranges)
+
+  intersection : (range_set) ->
+    new_ranges = []
+    for their_range in range_set.ranges
+      for our_range in @ranges
+        if @does_overlap(their_range, our_range)
+          if intrsc = @find_intersection(their_range, our_range)
+            new_ranges.push(intrsc)
+
+    new TwitterCldr.RangeSet new_ranges
+
+
+  subtract : (range_set) ->
+    return (new TwitterCldr.RangeSet(@ranges)) if range_set.is_empty()
+    remaining = TwitterCldr.Utilities.clone(range_set.ranges)
+    current_ranges = TwitterCldr.Utilities.clone(@ranges)
+
+    while their_range = remaining.shift()
+      new_ranges = []
+
+      for our_range in current_ranges
+        if @does_overlap(their_range, our_range)
+          new_ranges = new_ranges.concat(@find_subtraction(their_range, our_range))
+        else
+          new_ranges.push(our_range)
+
+      current_ranges = new_ranges
+
+    new TwitterCldr.RangeSet(new_ranges)
+
+
+  # symmetric difference (the union without the intersection)
+  # http://en.wikipedia.org/wiki/Symmetric_difference
+  difference : (range_set) ->
+    @union(range_set).subtract(@intersection(range_set))
+
+  flatten : ->
+    if @ranges.length <= 1
+      return
+
+    sorted_ranges = @ranges.sort ((a,b) ->
+      if ((!a.is_numeric()) and (!b.is_numeric()))
+        return 1
+      if a.first > b.first
+        return 1
+      else if a.first < b.first
+        return -1
+      else
+        return 0
+    )
+
+    new_ranges = [sorted_ranges[0]]
+    for range in sorted_ranges
+      previous_range = new_ranges.pop()
+
+      if (@are_adjacent(previous_range, range)) or (@does_overlap(previous_range, range))
+        new_ranges.push(new TwitterCldr.Range(TwitterCldr.Utilities.min([range.first, previous_range.first]),TwitterCldr.Utilities.max([range.last, previous_range.last])))
+      else
+        new_ranges.push(previous_range)
+        new_ranges.push(range)
+
+    @ranges = new_ranges
+
+  # returns true if range1 and range2 are within 1 of each other
+  are_adjacent : (range1, range2) ->
+    range1.is_numeric() and range2.is_numeric() and
+    ((range1.last is range2.first - 1) or (range2.first is range1.last + 1))
+
+  does_overlap : (range1, range2) ->
+    range1.is_numeric() and range2.is_numeric() and
+    (
+      (range1.last >= range2.first and range1.last <= range2.last) or
+      (range1.first >= range2.first and range1.first <= range2.last) or
+      (range1.first <= range2.first and range1.last >= range2.last)
+    )
+
+  find_intersection : (range1, range2) ->
+    # range2 entirely contains range1
+    if range2.first <= range1.first and range1.last <= range2.last
+      TwitterCldr.Utilities.clone(range1)
+    else if range1.last >= range2.first and range1.last <= range2.last
+      new TwitterCldr.Range(range2.first, range1.last)
+    else if range1.first >= range2.first and range1.first <= range2.last
+      new TwitterCldr.Range(range1.first, range2.last)
+    else if range1.first <= range2.first and range1.last >= range2.last
+      new TwitterCldr.Range(TwitterCldr.Utilities.max([range1.first, range2.first]),TwitterCldr.Utilities.min([range1.last, range2.last]))
+
+  # subtracts range1 from range2 (range2 - range1)
+  find_subtraction : (range1, range2) ->
+    # case: range1 contains range2 entirely (also handles equal case)
+    results = null
+    if range1.first <= range2.first and range2.last <= range1.last
+      results = []
+    # case: range1 comes in the middle
+    else if range2.first <= range1.first and range2.last >= range1.last
+      results = [(new TwitterCldr.Range(range2.first, range1.first - 1)), (new TwitterCldr.Range(range1.last + 1, range2.last))]
+    # case: range1 trails
+    else if range2.last >= range1.first and range1.last >= range2.last
+      results = [new TwitterCldr.Range(range2.first, range1.first - 1)]
+    # case: range1 leads
+    else if range1.last >= range2.first && range1.first <= range2.first
+      results = [new TwitterCldr.Range(range1.last + 1, range2.last)]
+
+    filtered_results = []
+    for range in results
+      if range.first <= range.last
+        filtered_results.push(range)
+
+    filtered_results