twitter_cldr 1.6.2 → 1.7.0

data/Gemfile

@@ -1,4 +1,4 @@
-source 'http://rubygems.org'
+source :rubygems
 
 gemspec
 
@@ -11,6 +11,8 @@ group :development do
   gem 'nokogiri'
   gem 'ruby_parser', '~> 2.3.1'
 
+  gem 'ruby-cldr', :github => 'KL-7/ruby-cldr', :branch => 'for-twitter-cldr'
+
   platform :ruby do
     gem 'therubyracer',  '~> 0.9.10'
     gem 'uglifier',      '~> 1.2.4'

data/History.txt

@@ -1,3 +1,11 @@
+=== 1.7.0
+
+* Wrote rake tasks to update CLDR and ICU resources.
+* All resource files now written with symbolized keys so the gem doesn't have to recursively symbolize them on load.
+* Unicode code points now represented internally with integers instead of strings for better performance.
+* Added number formatting in JavaScript.
+* Added telephone code lookup functionality (per country) and postal code validation.
+
 === 1.6.2
 
 * Collation tries now loaded from marshal dumps, collation running time improved by ~80%.

data/README.md

@@ -271,6 +271,61 @@ TwitterCldr::Shared::Languages.translate_language("chino tradicional", :es, :en)
 TwitterCldr::Shared::Languages.translate_language("Traditional Chinese", :en, :es)  # "chino tradicional"
 ```
 
+### Postal Codes
+
+The CLDR contains postal code validation regexes for a number of countries.
+
+```ruby
+# United States
+TwitterCldr::Shared::PostalCodes.valid?(:us, "94103")     # true
+TwitterCldr::Shared::PostalCodes.valid?(:us, "9410")      # false
+
+# England (Great Britain)
+TwitterCldr::Shared::PostalCodes.valid?(:gb, "BS98 1TL")  # true
+
+# Sweden
+TwitterCldr::Shared::PostalCodes.valid?(:se, "280 12")    # true
+
+# Canada
+TwitterCldr::Shared::PostalCodes.valid?(:ca, "V3H 1Z7")   # true
+```
+
+Get a list of supported territories by using the `#territories` method:
+
+```ruby
+TwitterCldr::Shared::PostalCodes.territories  # [:ve, :iq, :cx, :cv, ...]
+```
+
+Just want the regex?  No problem:
+
+```ruby
+TwitterCldr::Shared::PostalCodes.regex_for_territory(:us)  # /\d{5}([ \-]\d{4})?/
+```
+
+### Phone Codes
+
+Look up phone codes by territory:
+
+```ruby
+# United States
+TwitterCldr::Shared::PhoneCodes.code_for_territory(:us)  # "1"
+
+# Perú
+TwitterCldr::Shared::PhoneCodes.code_for_territory(:pe)  # "51"
+
+# Egypt
+TwitterCldr::Shared::PhoneCodes.code_for_territory(:eg)  # "20"
+
+# Denmark
+TwitterCldr::Shared::PhoneCodes.code_for_territory(:dk)  # "45"
+```
+
+Get a list of supported territories by using the `#territories` method:
+
+```ruby
+TwitterCldr::Shared::PhoneCodes.territories  # [:zw, :an, :tr, :by, :mh, ...]
+```
+
 ### Unicode Data
 
 TwitterCLDR provides ways to retrieve individual code points as well as normalize and decompose Unicode text.
@@ -278,7 +333,7 @@ TwitterCLDR provides ways to retrieve individual code points as well as normaliz
 Retrieve data for code points:
 
 ```ruby
-code_point = TwitterCldr::Shared::CodePoint.for_hex("1F3E9")
+code_point = TwitterCldr::Shared::CodePoint.find(0x1F3E9)
 code_point.name             # "LOVE HOTEL"
 code_point.bidi_mirrored    # "N"
 code_point.category         # "So"
@@ -288,13 +343,13 @@ code_point.combining_class  # "0"
 Convert characters to code points:
 
 ```ruby
-TwitterCldr::Utils::CodePoints.from_string("¿")  # ["00BF"]
+TwitterCldr::Utils::CodePoints.from_string("¿")  # [0xBF]
 ```
 
 Convert code points to characters:
 
 ```ruby
-TwitterCldr::Utils::CodePoints.to_string(["00BF"])  # "¿"
+TwitterCldr::Utils::CodePoints.to_string([0xBF])  # "¿"
 ```
 
 Normalize/decompose a Unicode string (NFD, NFKD, NFC, and NFKC implementations available).  Note that the normalized string will almost always look the same as the original string because most character display systems automatically combine decomposed characters.
@@ -306,29 +361,29 @@ TwitterCldr::Normalization::NFD.normalize("français")  # "français"
 Normalization is easier to see in hex:
 
 ```ruby
-# ["0065", "0073", "0070", "0061", "00F1", "006F", "006C"]
+# [101, 115, 112, 97, 241, 111, 108]
 TwitterCldr::Utils::CodePoints.from_string("español")
 
-# ["0065", "0073", "0070", "0061", "006E", "0303", "006F", "006C"]
+# [101, 115, 112, 97, 110, 771, 111, 108]
 TwitterCldr::Utils::CodePoints.from_string(TwitterCldr::Normalization::NFD.normalize("español"))
 ```
 
-Notice in the example above that the letter "ñ" was transformed from `00F1` to `006E 0303`, which represent the "n" and the "˜" respectively.
+Notice in the example above that the letter "ñ" was transformed from `241` to `110 771`, which represent the "n" and the "˜" respectively.
 
 A few convenience methods also exist for `String` that make it easy to normalize and get code points for strings:
 
 ```ruby
-# ["0065", "0073", "0070", "0061", "00F1", "006F", "006C"]
+# [101, 115, 112, 97, 241, 111, 108]
 "español".localize.code_points
 
-# ["0065", "0073", "0070", "0061", "006E", "0303", "006F", "006C"]
+# [101, 115, 112, 97, 110, 771, 111, 108]
 "español".localize.normalize.code_points
 ```
 
 Specify a specific normalization algorithm via the `:using` option.  NFD, NFKD, NFC, and NFKC algorithms are all supported (default is NFD):
 
 ```ruby
-# ["0065", "0073", "0070", "0061", "006E", "0303", "006F", "006C"]
+# [101, 115, 112, 97, 110, 771, 111, 108]
 "español".localize.normalize(:using => :NFKD).code_points
 ```
 
@@ -362,11 +417,6 @@ collator.get_collation_elements("Älg")   # [[39, 5, 143], [0, 157, 5], [61, 5,
 collator.get_sort_key("Älg")             # [39, 61, 51, 1, 134, 157, 6, 1, 143, 7]
 ```
 
-**Note**: The collation algorithm in TwitterCLDR isn't quite complete.  Here's what we still have to do:
-
-1.  Backwards accent sorting for French.
-2.  Uppercase-first sorting for languages like Danish.
-
 **Note**: The TwitterCLDR collator does not currently pass all the collation tests provided by Unicode, but for some strange reasons.  See the [summary](https://gist.github.com/f4ee3bd280a2257c5641) of these discrepancies if you're curious.
 
 ## About Twitter-specific Locales

data/Rakefile

@@ -42,21 +42,71 @@ end
 
 namespace :resources do
   namespace :update do
+
+    desc 'Import locales resources'
+    task :locales_resources, :cldr_path do |_, args|
+      TwitterCldr::Resources::LocalesResourcesImporter.new(
+          args[:cldr_path] || '../cldr',
+          './resources/locales'
+      ).import
+    end
+
+    desc 'Import custom locales resources'
+    task :custom_locales_resources do
+      TwitterCldr::Resources::CustomLocalesResourcesImporter.new('./resources/custom/locales').import
+    end
+
     desc 'Import tailoring resources from CLDR data (should be executed using JRuby 1.7 in 1.9 mode)'
-    task :tailoring do
-      importer = TwitterCldr::Resources::TailoringImporter.new(
-          ENV.fetch('CLDR_DATA_PATH', '../cldr-tailoring/'),
+    task :tailoring, :cldr_path, :icu4j_jar_path do |_, args|
+      TwitterCldr::Resources::TailoringImporter.new(
+          args[:cldr_path] || '../cldr',
           './resources/collation/tailoring',
-          ENV.fetch('ICU4J_JAR_PATH', '../icu4j-49_1.jar')
-      )
+          args[:icu4j_jar_path] ||'../icu4j-49_1.jar'
+      ).import(TwitterCldr.supported_locales)
+    end
+
+    desc 'Import Unicode data resources'
+    task :unicode_data, :unicode_data_path do |_, args|
+      TwitterCldr::Resources::UnicodeDataImporter.new(
+          args[:unicode_data_path] || '../unicode-data',
+          './resources/unicode_data'
+      ).import
+    end
 
-      TwitterCldr.supported_locales.each { |locale| importer.import(locale) }
+    desc 'Import composition exclusions resource'
+    task :composition_exclusions, :derived_normalization_props_path do |_, args|
+      TwitterCldr::Resources::CompositionExclusionsImporter.new(
+          args[:derived_normalization_props_path] || '../unicode-data/DerivedNormalizationProps.txt',
+          './resources/unicode_data'
+      ).import
     end
 
-    desc 'Updates default and tailoring tries dumps'
+    desc 'Import postal codes resource'
+    task :postal_codes, :cldr_path do |_, args|
+      TwitterCldr::Resources::PostalCodesImporter.new(
+          args[:cldr_path] || '../cldr',
+          './resources/shared'
+      ).import
+    end
+
+    desc 'Import phone codes resource'
+    task :phone_codes, :cldr_path do |_, args|
+      TwitterCldr::Resources::PhoneCodesImporter.new(
+          args[:cldr_path] || '../cldr',
+          './resources/shared'
+      ).import
+    end
+
+    desc 'Update default and tailoring tries dumps'
     task :tries do
       TwitterCldr::Resources::TriesDumper.update_dumps
     end
+
+    desc 'Update canonical compositions resource'
+    task :canonical_compositions do
+      TwitterCldr::Resources::CanonicalCompositionsUpdater.new('./resources/unicode_data').update
+    end
+
   end
 end
 

data/js/lib/compiler.rb

@@ -32,7 +32,9 @@ module TwitterCldr
         @renderers ||= {
           :plural_rules => TwitterCldr::Js::Renderers::PluralRules::PluralRulesRenderer,
           :timespan => TwitterCldr::Js::Renderers::Calendars::TimespanRenderer,
-          :datetime => TwitterCldr::Js::Renderers::Calendars::DateTimeRenderer
+          :datetime => TwitterCldr::Js::Renderers::Calendars::DateTimeRenderer,
+          :numbers => TwitterCldr::Js::Renderers::Numbers::NumbersRenderer,
+          :currencies => TwitterCldr::Js::Renderers::Shared::CurrenciesRenderer
         }
       end
     end

data/js/lib/mustache/bundle.coffee

@@ -3,11 +3,11 @@
 // http://www.apache.org/licenses/LICENSE-2.0
 
 // TwitterCLDR (JavaScript) v{{version}}
-// Authors: 		Cameron Dutro [@camertron]
-								Kirill Lashuk [@KL_7]
-								portions by Sven Fuchs [@svenfuchs]
-// Homepage: 		https://twitter.com
-// Description:	Provides date, time, number, and list formatting functionality for various Twitter-supported locales in Javascript.
+// Authors:     Cameron Dutro [@camertron]
+                Kirill Lashuk [@KL_7]
+                portions by Sven Fuchs [@svenfuchs]
+// Homepage:    https://twitter.com
+// Description: Provides date, time, number, and list formatting functionality for various Twitter-supported locales in Javascript.
 ###
 
 TwitterCldr = {}

data/js/lib/mustache/numbers/numbers.coffee

@@ -0,0 +1,179 @@
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+TwitterCldr.NumberFormatter = class NumberFormatter
+	constructor: ->
+    @all_tokens = `{{{tokens}}}`
+    @tokens = []
+    @symbols = `{{{symbols}}}`
+
+    @default_symbols = 
+      'group': ','
+      'decimal': '.'
+      'plus_sign': '+'
+      'minus_sign': '-'
+
+  format: (number, options = {}) ->
+    opts = this.default_format_options_for(number)
+
+    for key, val of options
+      opts[key] = if options[key]? then options[key] else opts[key]
+
+    [prefix, suffix, integer_format, fraction_format] = this.partition_tokens(this.get_tokens(number, opts))
+    [int, fraction] = this.parse_number(number, opts)
+    result = integer_format.apply(parseFloat(int), opts)
+    result += fraction_format.apply(fraction, opts) if fraction
+    sign = if number < 0 && prefix != "-" then @symbols.minus_sign || @default_symbols.minus_sign else ""
+    "#{sign}#{prefix}#{result}#{suffix}"
+
+  partition_tokens: (tokens) ->
+    [tokens[0] || "",
+     tokens[2] || "",
+     new IntegerHelper(tokens[1], @symbols),
+     new FractionHelper(tokens[1], @symbols)]
+
+  parse_number: (number, options = {}) ->
+    if options.precision?
+      precision = options.precision
+    else
+      precision = this.precision_from(number)
+
+    number = this.round_to(number, precision)
+    Math.abs(number).toFixed(precision).split(".")
+
+  precision_from: (num) ->
+    parts = num.toString().split(".")
+    if parts.length == 2 then parts[1].length else 0
+
+  round_to: (number, precision) ->
+    factor = Math.pow(10, precision)
+    Math.round(number * factor) / factor
+
+  get_tokens: ->
+    throw "get_tokens() not implemented - use a derived class like PercentFormatter."
+
+TwitterCldr.PercentFormatter = class PercentFormatter extends NumberFormatter
+  constructor: (options = {}) ->
+    @default_percent_sign = "%"
+    super
+
+  format: (number, options = {}) ->
+    super(number, options).replace('¤', @symbols.percent_sign || @default_percent_sign)
+
+  default_format_options_for: (number) ->
+    precision: 0
+
+  get_tokens: (number, options) ->
+    if number < 0 then @all_tokens.percent.negative else @all_tokens.percent.positive
+
+TwitterCldr.DecimalFormatter = class DecimalFormatter extends NumberFormatter
+  format: (number, options = {}) ->
+    try
+      super(number, options)
+    catch error
+      number
+
+  default_format_options_for: (number) ->
+    precision: this.precision_from(number)
+
+  get_tokens: (number, options = {}) ->
+    if number < 0 then @all_tokens.decimal.negative else @all_tokens.decimal.positive
+
+TwitterCldr.CurrencyFormatter = class CurrencyFormatter extends NumberFormatter
+  constructor: (options = {}) ->
+    @default_currency_symbol = "$"
+    @default_precision = 2
+    super
+
+  format: (number, options = {}) ->
+    if options.currency
+      if TwitterCldr.Currencies?
+        currency = TwitterCldr.Currencies.for_code(options.currency)
+        currency ||= TwitterCldr.Currencies.for_country(options.currency)
+        currency ||= symbol: options.currency
+      else
+        currency = symbol: options.currency
+    else
+      currency = symbol: @default_currency_symbol
+
+    super(number, options).replace('¤', currency.symbol)
+
+  default_format_options_for: (number) ->
+    precision = this.precision_from(number)
+    if precision == 0 then precision = @default_precision
+    precision: precision
+
+  get_tokens: (number, options = {}) ->
+    if number < 0 then @all_tokens.currency.negative else @all_tokens.currency.positive
+
+TwitterCldr.NumberFormatter.BaseHelper = class BaseHelper
+  interpolate: (string, value, orientation = "right") ->
+    value = value.toString()
+    length = value.length
+    start = if orientation == "left" then 0 else -length
+    string = (("#" for i in [0...length]).join("") + string).slice(-length) if string.length < length
+
+    if start < 0
+      string = string[0...(start + string.length)] + value
+    else
+      string = string[0...start] + value + string[(length)..-1]
+
+    string.replace(/#/g, "")
+
+TwitterCldr.NumberFormatter.IntegerHelper = class IntegerHelper extends BaseHelper
+  constructor: (token, symbols = {}) ->
+    format     = token.split('.')[0]
+    @format    = this.prepare_format(format, symbols)
+    @groups    = this.parse_groups(format)
+    @separator = symbols.group || ','
+
+  apply: (number, options = {}) ->
+    this.format_groups(this.interpolate(@format, parseInt(number)))
+
+  format_groups: (string) ->
+    return string if @groups.length == 0
+    tokens = []
+
+    cur_token = this.chop_group(string, @groups[0])
+    tokens.push(cur_token)
+    string = string[0...(string.length - cur_token.length)] if cur_token
+
+    while string.length > @groups[@groups.length - 1]
+      cur_token = this.chop_group(string, @groups[@groups.length - 1])
+      tokens.push(cur_token)
+      string = string[0...(string.length - cur_token.length)] if cur_token
+
+    tokens.push(string)
+    (token for token in tokens when token != null).reverse().join(@separator)
+
+  parse_groups: (format) ->
+    return [] unless index = format.lastIndexOf(',')
+    rest = format[0...index]
+    widths = [format.length - index - 1]
+    widths.push(rest.length - rest.lastIndexOf(',') - 1) if rest.lastIndexOf(',') > -1
+    widths = (width for width in widths when width != null)  # compact
+    widths.reverse()  # uniq
+    (widths[index] for index in [0...widths.length] when widths.indexOf(widths[index], index + 1) == -1).reverse()
+
+  chop_group: (string, size) ->
+    if string.length > size then string[-size..-1] else null
+
+  prepare_format: (format, symbols) ->
+    format.replace(",", "").replace("+", symbols.plus_sign).replace("-", symbols.minus_sign)
+
+TwitterCldr.NumberFormatter.FractionHelper = class FractionHelper extends BaseHelper
+  constructor: (token, symbols = {}) ->
+    @format = if token then token.split('.').pop() else ""
+    @decimal = symbols.decimal || "."
+    @precision = @format.length
+
+  apply: (fraction, options = {}) ->
+    precision = if options.precision? then options.precision else @precision
+    if precision > 0
+      @decimal + this.interpolate(this.format_for(options), fraction, "left")
+    else
+      ""
+
+  format_for: (options) ->
+    precision = if options.precision? then options.precision else @precision
+    if precision then ("0" for i in [0...precision]).join("") else @format