twitter_cldr 3.0.0.beta1 → 3.0.0

data/Rakefile

@@ -61,70 +61,114 @@ else
   end
 end
 
+task :update do
+  tasks = [
+    "update:locales_resources",
+    "update:tailoring_data",
+    "update:unicode_data",
+    "update:unicode_properties",
+    "update:generate_casefolder",  # must come after unicode data
+    "update:composition_exclusions",
+    "update:postal_codes",
+    "update:phone_codes",
+    "update:language_codes",
+    "update:collation_tries",
+    "update:canonical_compositions",
+    "update:rbnf_tests",
+    "update:segment_exceptions",
+    "update:readme"
+  ]
+
+  tasks.each do |task|
+    puts "Executing #{task}"
+    Rake::Task[task].invoke
+  end
+end
+
 namespace :update do
+  ICU_JAR = './vendor/icu4j-51_2.jar'
+
   desc 'Import locales resources'
   task :locales_resources, :cldr_path do |_, args|
     TwitterCldr::Resources::LocalesResourcesImporter.new(
-        args[:cldr_path] || './vendor/cldr',
-        './resources'
+      args[:cldr_path] || './vendor/cldr',
+      './resources'
     ).import
   end
 
   desc 'Import custom locales resources'
   task :custom_locales_resources do
-    TwitterCldr::Resources::CustomLocalesResourcesImporter.new('./resources/custom/locales').import
+    TwitterCldr::Resources::CustomLocalesResourcesImporter.new(
+      './resources/custom/locales'
+    ).import
   end
 
   desc 'Import tailoring resources from CLDR data (should be executed using JRuby 1.7 in 1.9 mode)'
   task :tailoring_data, :cldr_path, :icu4j_jar_path do |_, args|
     TwitterCldr::Resources::TailoringImporter.new(
-        args[:cldr_path] || './vendor/cldr',
-        './resources/collation/tailoring',
-        args[:icu4j_jar_path] || './vendor/icu4j-51_2.jar'
+      args[:cldr_path] || './vendor/cldr',
+      './resources/collation/tailoring',
+      args[:icu4j_jar_path] || ICU_JAR
     ).import(TwitterCldr.supported_locales)
   end
 
   desc 'Import Unicode data resources'
   task :unicode_data, :unicode_data_path do |_, args|
     TwitterCldr::Resources::UnicodeDataImporter.new(
-        args[:unicode_data_path] || './vendor/unicode-data',
-        './resources/unicode_data'
+      args[:unicode_data_path] || './vendor/unicode-data',
+      './resources/unicode_data'
     ).import
   end
 
+  desc 'Import Unicode property resources'
+  task :unicode_properties, :unicode_properties_path do |_, args|
+    TwitterCldr::Resources::UnicodePropertiesImporter.new(
+      args[:unicode_properties_path] || './vendor/unicode-data/properties',
+      './resources/unicode_data/properties'
+    ).import
+  end
+
+  desc 'Generate the casefolder class. Depends on unicode data'
+  task :generate_casefolder do
+    TwitterCldr::Resources::CasefolderClassGenerator.new(
+      './lib/twitter_cldr/resources/casefolder.rb.erb',
+      './lib/twitter_cldr/shared'
+    ).generate
+  end
+
   desc 'Import composition exclusions resource'
   task :composition_exclusions, :derived_normalization_props_path do |_, args|
     TwitterCldr::Resources::CompositionExclusionsImporter.new(
-        args[:derived_normalization_props_path] || './vendor/unicode-data/DerivedNormalizationProps.txt',
-        './resources/unicode_data'
+      args[:derived_normalization_props_path] || './vendor/unicode-data/DerivedNormalizationProps.txt',
+      './resources/unicode_data'
     ).import
   end
 
   desc 'Import postal codes resource'
   task :postal_codes, :cldr_path do |_, args|
     TwitterCldr::Resources::PostalCodesImporter.new(
-        args[:cldr_path] || './vendor/cldr',
-        './resources/shared'
+      args[:cldr_path] || './vendor/cldr',
+      './resources/shared'
     ).import
   end
 
   desc 'Import phone codes resource'
   task :phone_codes, :cldr_path do |_, args|
     TwitterCldr::Resources::PhoneCodesImporter.new(
-        args[:cldr_path] || './vendor/cldr',
-        './resources/shared'
+      args[:cldr_path] || './vendor/cldr',
+      './resources/shared'
     ).import
   end
 
   desc 'Import language codes'
   task :language_codes, :language_codes_data do |_, args|
     TwitterCldr::Resources::LanguageCodesImporter.new(
-        args[:language_codes_data] || './vendor/language-codes',
-        './resources/shared'
+      args[:language_codes_data] || './vendor/language-codes',
+      './resources/shared'
     ).import
   end
 
-  desc 'Update default and tailoring tries dumps'
+  desc 'Update default and tailoring tries dumps (should be executed using JRuby 1.7 in 1.9 mode)'
   task :collation_tries do
     TwitterCldr::Resources::CollationTriesDumper.update_dumps
   end
@@ -141,4 +185,40 @@ namespace :update do
       './resources/unicode_data'
     ).import
   end
+
+  desc 'Import (generate) bidi tests (should be executed using JRuby 1.7 in 1.9 mode)'
+  task :bidi_tests do |_, args|
+    TwitterCldr::Resources::BidiTestImporter.new(
+      './spec/bidi'
+    ).import
+  end
+
+  desc 'Import (generate) rule-based number format tests (should be executed using JRuby 1.7 in 1.9 mode)'
+  task :rbnf_tests, :icu4j_jar_path do |_, args|
+    TwitterCldr::Resources::RbnfTestImporter.new(
+      './spec/formatters/numbers/rbnf/locales',
+      args[:icu4j_jar_path] || ICU_JAR
+    ).import(TwitterCldr.supported_locales)
+  end
+
+  task :segment_exceptions do
+    TwitterCldr::Resources::Uli::SegmentExceptionsImporter.new(
+      './vendor/uli/segments',
+      './resources/uli/segments'
+    ).import([:de, :en, :es, :fr, :it, :pt, :ru])  # only locales ULI supports at the moment
+  end
+
+  task :readme do |_, args|
+    renderer = TwitterCldr::Resources::ReadmeRenderer.new(
+      File.read("./README.md.erb")
+    )
+
+    File.open("./README.md", "w+") do |f|
+      f.write(renderer.render)
+    end
+
+    if renderer.assertion_failures.size > 0
+      puts "There were errors encountered while updating the README. Please run specs for details"
+    end
+  end
 end

data/lib/twitter_cldr/collation/collator.rb

@@ -57,12 +57,21 @@ module TwitterCldr
       end
 
       def get_normalized_code_points(str_or_code_points)
-        code_points = str_or_code_points.is_a?(String) ? TwitterCldr::Utils::CodePoints.from_string(str_or_code_points) : str_or_code_points
+        str = if str_or_code_points.is_a?(String)
+          str_or_code_points
+        else
+          TwitterCldr::Utils::CodePoints.to_string(str_or_code_points)
+        end
 
         # Normalization makes the collation process significantly slower (like seven times slower on the UCA
         # non-ignorable test from CollationTest_NON_IGNORABLE.txt). ICU uses some optimizations to apply normalization
         # only in special, rare cases. We need to investigate possible solutions and do normalization cleverly too.
-        TwitterCldr::Normalization::NFD.normalize_code_points(code_points)
+        TwitterCldr::Utils::CodePoints.from_string(
+          TwitterCldr::Normalization.normalize(str)
+        )
+      rescue ArgumentError
+        # Meant to rescue encoding errors. Some tests do not provide valid UTF-8 sequences.
+        TwitterCldr::Utils::CodePoints.from_string(str)
       end
 
       # Returns the first sequence of fractional collation elements for an array of integer code points. Returned value
@@ -99,7 +108,7 @@ module TwitterCldr
         while non_starter_pos < code_points.size && !suffixes.empty?
           # get next code point (possibly non-starter)
           non_starter_code_point = code_points[non_starter_pos]
-          combining_class        = TwitterCldr::Normalization::Base.combining_class_for(non_starter_code_point)
+          combining_class        = TwitterCldr::Shared::CodePoint.find(non_starter_code_point).combining_class.to_i
 
           # code point is a starter or combining class has been already used (non-starter is 'blocked' from the prefix)
           break if combining_class == 0 || used_combining_classes[combining_class]

data/lib/twitter_cldr/collation/trie_builder.rb

@@ -31,7 +31,7 @@ module TwitterCldr
         private
 
         def parse_collation_elements_table(table, trie = TwitterCldr::Collation::Trie.new)
-          table.lines.each do |line|
+          table.each_line do |line|
             trie.set(parse_code_points($1), parse_collation_element($2)) if COLLATION_ELEMENT_REGEXP =~ line
           end
 

data/lib/twitter_cldr/{tokenizers/calendars → data_readers}/additional_date_format_selector.rb

@@ -4,7 +4,7 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 
 module TwitterCldr
-  module Tokenizers
+  module DataReaders
     class AdditionalDateFormatSelector
       attr_reader :pattern_hash
 
@@ -18,17 +18,19 @@ module TwitterCldr
         else
           cache_key = TwitterCldr::Utils.compute_cache_key(goal_pattern)
           pattern_cache[cache_key] ||= if @pattern_hash.include?(goal_pattern.to_sym)
-            goal_pattern
+            goal_pattern.to_sym
           else
             rank(goal_pattern).min do |(p1, score1), (p2, score2)|
               score1 <=> score2
-            end.first
+            end.first.to_sym
           end
+
+          @pattern_hash[pattern_cache[cache_key]]
         end
       end
 
       def patterns
-        @pattern_hash.keys
+        @pattern_hash.keys.map(&:to_s)
       end
 
       protected

data/lib/twitter_cldr/data_readers/calendar_data_reader.rb

@@ -0,0 +1,91 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module DataReaders
+    class CalendarDataReader < DataReader
+
+      DEFAULT_TYPE = :medium
+
+      TYPE_PATHS = {
+        :full       => [:full, :pattern],
+        :long       => [:long, :pattern],
+        :medium     => [:medium, :pattern],
+        :short      => [:short, :pattern],
+        :additional => [:additional_formats]
+      }
+
+      class << self
+        def types
+          TYPE_PATHS.keys
+        end
+      end
+
+      attr_reader :calendar_type, :type, :additional_format
+
+      def initialize(locale, options = {})
+        super(locale)
+        @calendar_type = options[:calendar_type] || TwitterCldr::DEFAULT_CALENDAR_TYPE
+        @type = options[:type] || type || :default
+        @type = DEFAULT_TYPE if type == :default
+        @additional_format = options[:additional_format]
+      end
+
+      def pattern
+        if type == :additional
+          additional_format_selector.find_closest(additional_format)
+        else
+          traverse(path_for(type, calendar_type) + TYPE_PATHS[type])
+        end
+      end
+
+      def calendar
+        @calendar ||= TwitterCldr::Shared::Calendar.new(locale)
+      end
+
+      def additional_format_selector
+        @format_selector ||= AdditionalDateFormatSelector.new(
+          traverse([:calendars, calendar_type, :additional_formats])
+        )
+      end
+
+      protected
+
+      def path_for(type, calendar_type)
+        raise NotImplementedError
+      end
+
+      def resource
+        @resource ||= begin
+          resource = TwitterCldr.get_locale_resource(locale, :calendars)[locale]
+          resource[:calendars].each_pair do |calendar_type, options|
+            next if calendar_type == TwitterCldr::DEFAULT_CALENDAR_TYPE
+            mirror_resource(
+              :from => resource[:calendars][TwitterCldr::DEFAULT_CALENDAR_TYPE],
+              :to   => resource[:calendars][calendar_type]
+            )
+          end
+          resource
+        end
+      end
+
+      def mirror_resource(options)
+        from = options[:from]
+        to = options[:to]
+
+        from.each_pair do |key, value|
+          if !to[key]
+            to[key] = from[key]
+          else
+            if to[key].is_a?(Hash) and from[key].is_a?(Hash)
+              mirror_resource(:from => from[key], :to => to[key])
+            end
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/data_readers/data_reader.rb

@@ -0,0 +1,32 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module DataReaders
+    class DataReader
+
+      attr_reader :locale
+
+      def initialize(locale)
+        @locale = locale
+      end
+
+      def pattern_at_path(path)
+        traverse(path)
+      end
+
+      private
+
+      def traverse(path, hash = resource)
+        TwitterCldr::Utils.traverse_hash(hash, path)
+      end
+
+      def resource
+        raise NotImplementedError
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/data_readers/date_data_reader.rb

@@ -0,0 +1,26 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module DataReaders
+    class DateDataReader < CalendarDataReader
+
+      def tokenizer
+        @tokenizer ||= TwitterCldr::Tokenizers::DateTokenizer.new(self)
+      end
+
+      def formatter
+        @formatter ||= TwitterCldr::Formatters::DateTimeFormatter.new(self)
+      end
+
+      protected
+
+      def path_for(type, calendar_type)
+        [:calendars, calendar_type, :formats, :date]
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/data_readers/date_time_data_reader.rb

@@ -0,0 +1,41 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module DataReaders
+    class DateTimeDataReader < CalendarDataReader
+
+      def date_reader
+        @date_reader ||= DateDataReader.new(locale, gather_options)
+      end
+
+      def time_reader
+        @time_reader ||= TimeDataReader.new(locale, gather_options)
+      end
+
+      def tokenizer
+        @tokenizer ||= TwitterCldr::Tokenizers::DateTimeTokenizer.new(self)
+      end
+
+      def formatter
+        @formatter ||= TwitterCldr::Formatters::DateTimeFormatter.new(self)
+      end
+
+      protected
+
+      def gather_options
+        {
+          :type => type,
+          :calendar_type => calendar_type
+        }
+      end
+
+      def path_for(type, calendar_type)
+        [:calendars, calendar_type, :formats, :datetime]
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/data_readers/number_data_reader.rb

@@ -0,0 +1,142 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module DataReaders
+    class NumberDataReader < DataReader
+
+      DEFAULT_NUMBER_SYSTEM = "latn"
+      ABBREVIATED_MIN_POWER = 3
+      ABBREVIATED_MAX_POWER = 14
+
+      NUMBER_MAX = 10 ** (ABBREVIATED_MAX_POWER + 1)
+      NUMBER_MIN = 10 ** ABBREVIATED_MIN_POWER
+
+      BASE_PATH   = [:numbers, :formats]
+      SYMBOL_PATH = [:numbers, :symbols]
+
+      TYPE_PATHS = {
+        :default       => [:decimal, :patterns],
+        :decimal       => [:decimal, :patterns],
+        :long_decimal  => [:decimal, :patterns, :long],
+        :short_decimal => [:decimal, :patterns, :short],
+        :currency      => [:currency, :patterns],
+        :percent       => [:percent, :patterns]
+      }
+
+      ABBREVIATED_TYPES = [:long_decimal, :short_decimal]
+
+      DEFAULT_TYPE = :decimal
+      DEFAULT_FORMAT = :default
+      DEFAULT_SIGN = :positive
+
+      attr_reader :type, :format
+
+      def initialize(locale, options = {})
+        super(locale)
+        @type = options[:type] || DEFAULT_TYPE
+
+        unless type && TYPE_PATHS.include?(type.to_sym)
+          raise ArgumentError.new("Type #{type} is not supported")
+        end
+
+        @format = options[:format] || DEFAULT_FORMAT
+      end
+
+      def pattern(number = nil)
+        sign = number < 0 ? :negative : :positive 
+        path = BASE_PATH + if valid_type_for?(number, type)
+          TYPE_PATHS[type]
+        else
+          TYPE_PATHS[:default]
+        end
+
+        pattern = traverse(path)
+
+        if pattern[format]
+          pattern = pattern[format]
+        end
+
+        if number
+          pattern = pattern_for_number(pattern, number)
+        end
+
+        if pattern.is_a?(String)
+          pattern_for_sign(pattern, sign)
+        else
+          pattern
+        end
+      end
+
+      def number_system_for(type)
+        (traverse(BASE_PATH + [type]) || {}).fetch(:number_system, DEFAULT_NUMBER_SYSTEM)
+      end
+
+      def symbols
+        @symbols ||= traverse(SYMBOL_PATH)
+      end
+
+      def tokenizer
+        @tokenizer ||= TwitterCldr::Tokenizers::NumberTokenizer.new(self)
+      end
+
+      def formatter
+        @formatter ||= begin
+          klass_name = type.to_s.split("_").map(&:capitalize).join
+          klass = TwitterCldr::Formatters.const_get(:"#{klass_name}Formatter")
+          klass.new(self)
+        end
+      end
+
+      private
+
+      def abbreviated?(type)
+        ABBREVIATED_TYPES.include?(type)
+      end
+
+      def valid_type_for?(number, type)
+        if abbreviated?(type)
+          (number < NUMBER_MAX) && (number >= NUMBER_MIN)
+        else
+          TYPE_PATHS.include?(type)
+        end
+      end
+
+      def get_key_for(number)
+        "1#{"0" * (number.to_i.to_s.size - 1)}".to_i
+      end
+
+      def pattern_for_number(pattern, number)
+        if pattern.is_a?(Hash)
+          pattern[get_key_for(number)] || pattern
+        else
+          pattern
+        end
+      end
+
+      def pattern_for_sign(pattern, sign)
+        if pattern.include?(";")
+          positive, negative = pattern.split(";")
+          sign == :positive ? positive : negative
+        else
+          case sign
+            when :negative
+              "#{symbols[:minus] || '-'}#{pattern}"
+            else
+              pattern
+          end
+        end
+      end
+
+      def resource
+        @resource ||= begin
+          raw = TwitterCldr.get_locale_resource(locale, :numbers)
+          raw[TwitterCldr.convert_locale(locale)]
+        end
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/data_readers/time_data_reader.rb

@@ -0,0 +1,26 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module DataReaders
+    class TimeDataReader < CalendarDataReader
+
+      def tokenizer
+        @tokenizer ||= TwitterCldr::Tokenizers::TimeTokenizer.new(self)
+      end
+
+      def formatter
+        @formatter ||= TwitterCldr::Formatters::DateTimeFormatter.new(self)
+      end
+
+      protected
+
+      def path_for(type, calendar_type)
+        [:calendars, calendar_type, :formats, :time]
+      end
+
+    end
+  end
+end