twitter_cldr 1.7.0 → 1.8.0

data/lib/twitter_cldr/resources/locales_resources_importer.rb

@@ -5,7 +5,7 @@
 
 require 'cldr/export'
 
-require 'lib/twitter_cldr/resources/download'
+require 'twitter_cldr/resources/download'
 
 module TwitterCldr
   module Resources

data/lib/twitter_cldr/resources/phone_codes_importer.rb

@@ -5,7 +5,7 @@
 
 require 'nokogiri'
 
-require 'lib/twitter_cldr/resources/download'
+require 'twitter_cldr/resources/download'
 
 module TwitterCldr
   module Resources

data/lib/twitter_cldr/resources/postal_codes_importer.rb

@@ -5,7 +5,7 @@
 
 require 'nokogiri'
 
-require 'lib/twitter_cldr/resources/download'
+require 'twitter_cldr/resources/download'
 
 module TwitterCldr
   module Resources

data/lib/twitter_cldr/resources/tailoring_importer.rb

@@ -6,7 +6,7 @@
 require 'nokogiri'
 require 'java'
 
-require 'lib/twitter_cldr/resources/download'
+require 'twitter_cldr/resources/download'
 
 module TwitterCldr
   module Resources
@@ -34,6 +34,8 @@ module TwitterCldr
 
       EMPTY_TAILORING_DATA = { :collator_options => {}, :tailored_table => '', :suppressed_contractions => '' }
 
+      ICU4J_URL = 'http://download.icu-project.org/files/icu4j/49.1/icu4j-49_1.jar'
+
       class ImportError < RuntimeError; end
 
       # Arguments:
@@ -43,7 +45,7 @@ module TwitterCldr
       #   icu4j_path  - path to ICU4J jar file
       #
       def initialize(input_path, output_path, icu4j_path)
-        require icu4j_path
+        require_icu4j(icu4j_path)
 
         @input_path  = input_path
         @output_path = output_path
@@ -56,6 +58,11 @@ module TwitterCldr
 
       private
 
+      def require_icu4j(icu4j_path)
+        TwitterCldr::Resources.download_if_necessary(icu4j_path, ICU4J_URL)
+        require icu4j_path
+      end
+
       def import_locale(locale)
         print "Importing %8s\t--\t" % locale
 
@@ -126,7 +133,7 @@ module TwitterCldr
             end
           elsif child.name == 'x'
             context = ''
-            child.children.each_with_object([]) do |c, memo|
+            child.children.inject([]) do |memo, c|
               if SIMPLE_RULES.include?(c.name)
                 memo << table_entry_for_rule(collator, context + c.text)
               elsif c.name == 'context'
@@ -134,6 +141,8 @@ module TwitterCldr
               elsif c.name != 'extend'
                 raise ImportError, "Rule '#{c.name}' inside <x></x> is not supported."
               end
+
+              memo
             end
           else
             raise ImportError, "Tag '#{child.name}' is not supported." unless IGNORED_TAGS.include?(child.name)

data/lib/twitter_cldr/resources/unicode_data_importer.rb

@@ -3,7 +3,7 @@
 # Copyright 2012 Twitter, Inc
 # http://www.apache.org/licenses/LICENSE-2.0
 
-require 'lib/twitter_cldr/resources/download'
+require 'twitter_cldr/resources/download'
 
 module TwitterCldr
   module Resources
@@ -29,6 +29,8 @@ module TwitterCldr
 
         File.open(File.join(@output_path, 'blocks.yml'), 'w') { |output| YAML.dump(blocks, output) }
 
+        FileUtils.mkdir_p(File.join(@output_path, 'blocks'))
+
         unicode_data.each do |block_name, code_points|
           File.open(File.join(@output_path, 'blocks', "#{block_name}.yml"), 'w') { |output| YAML.dump(code_points, output) }
         end

data/lib/twitter_cldr/resources.rb

@@ -6,14 +6,15 @@
 module TwitterCldr
   module Resources
     autoload :CanonicalCompositionsUpdater,   'twitter_cldr/resources/canonical_compositions_updater'
+    autoload :CollationTriesDumper,           'twitter_cldr/resources/collation_tries_dumper'
     autoload :CompositionExclusionsImporter,  'twitter_cldr/resources/composition_exclusions_importer'
     autoload :CustomLocalesResourcesImporter, 'twitter_cldr/resources/custom_locales_resources_importer'
+    autoload :LanguageCodesImporter,          'twitter_cldr/resources/language_codes_importer'
     autoload :Loader,                         'twitter_cldr/resources/loader'
     autoload :LocalesResourcesImporter,       'twitter_cldr/resources/locales_resources_importer'
     autoload :PhoneCodesImporter,             'twitter_cldr/resources/phone_codes_importer'
     autoload :PostalCodesImporter,            'twitter_cldr/resources/postal_codes_importer'
     autoload :TailoringImporter,              'twitter_cldr/resources/tailoring_importer'
-    autoload :TriesDumper,                    'twitter_cldr/resources/tries_dumper'
     autoload :UnicodeDataImporter,            'twitter_cldr/resources/unicode_data_importer'
   end
 end

data/lib/twitter_cldr/shared/calendar.rb

@@ -34,15 +34,11 @@ module TwitterCldr
       end
 
       def get_data(*path)
-        data = traverse_path(calendar_data, path)
+        data = TwitterCldr::Utils.traverse_hash(calendar_data, path)
         redirect = parse_redirect(data)
         redirect ? get_data(*redirect) : data
       end
 
-      def traverse_path(tree, path)
-        !path.empty? && tree.is_a?(Hash) ? traverse_path(tree[path.first], path[1..-1]) : tree
-      end
-
       def parse_redirect(data)
         $1.split('.').map(&:to_sym) if data.is_a?(Symbol) && data.to_s =~ redirect_regexp
       end
@@ -52,7 +48,7 @@ module TwitterCldr
       end
 
       def calendar_data
-        @calendar_data ||= traverse_path(resource, [locale, :calendars, calendar_type])
+        @calendar_data ||= TwitterCldr::Utils.traverse_hash(resource, [locale, :calendars, calendar_type])
       end
 
       def resource

data/lib/twitter_cldr/shared/language_codes.rb

@@ -0,0 +1,75 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Shared
+    module LanguageCodes
+
+      LANGUAGE_CODES_DUMP_PATH = File.join(TwitterCldr::RESOURCES_DIR, 'shared', 'language_codes_table.dump')
+
+      NAME_STANDARD = :name # fake standard, mostly for internal use
+
+      VALID_STANDARDS = [:bcp_47, :iso_639_1, :iso_639_2, :iso_639_3, NAME_STANDARD]
+
+      class << self
+
+        def languages
+          resource[NAME_STANDARD].keys
+        end
+
+        def valid_standard?(standard)
+          VALID_STANDARDS.include?(standard.to_sym)
+        end
+
+        def valid_code?(code, standard)
+          resource[validate_standard(standard)].has_key?(code.to_sym)
+        end
+
+        def convert(code, from_and_to = {})
+          from, to = extract_from_and_to_options(from_and_to)
+          resource[from].fetch(code.to_sym, {})[to]
+        end
+
+        def from_language(language, standard)
+          convert(language, :from => NAME_STANDARD, :to => standard)
+        end
+
+        def to_language(code, standard)
+          convert(code, :from => standard, :to => NAME_STANDARD).to_s
+        end
+
+        def standards_for(code, standard)
+          resource[validate_standard(standard)].fetch(code.to_sym, {}).keys - [NAME_STANDARD] # exclude fake NAME_STANDARD standard
+        end
+
+        def standards_for_language(language)
+          standards_for(language, NAME_STANDARD)
+        end
+
+        private
+
+        def resource
+          @resource ||= File.open(LANGUAGE_CODES_DUMP_PATH) { |file| Marshal.load(file.read) }
+        end
+
+        def extract_from_and_to_options(from_and_to)
+          TwitterCldr::Utils.deep_symbolize_keys(from_and_to).values_at(:from, :to).map do |standard|
+            raise ArgumentError, 'options :from and :to are required' if standard.nil?
+            validate_standard(standard)
+          end
+        end
+
+        def validate_standard(standard)
+          raise ArgumentError, "standard can't be nil" if standard.nil?
+          raise ArgumentError, "#{standard.inspect} is not a valid standard name" unless valid_standard?(standard)
+
+          standard.to_sym
+        end
+
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/shared/languages.rb

@@ -14,7 +14,7 @@ module TwitterCldr
         end
 
         def all_for(code)
-          get_resource(TwitterCldr.convert_locale(code.to_sym))[:languages]
+          get_resource(code)[:languages]
         rescue
           {}
         end
@@ -24,21 +24,14 @@ module TwitterCldr
         end
 
         def from_code_for_locale(code, locale = TwitterCldr.get_locale)
-          get_resource(TwitterCldr.convert_locale(locale.to_sym))[:languages][TwitterCldr.convert_locale(code.to_sym)]
+          get_resource(locale)[:languages][TwitterCldr.convert_locale(code)]
         rescue
           nil
         end
 
         def translate_language(language, source_locale = :en, dest_locale = TwitterCldr.get_locale)
-          source_locale = TwitterCldr.convert_locale(source_locale.to_sym)
-          lang_code = get_resource(source_locale)[:languages].select { |key, val| val.downcase == language.downcase }.flatten.first
-
-          if lang_code
-            dest_locale = TwitterCldr.convert_locale(dest_locale.to_sym)
-            get_resource(dest_locale)[:languages][lang_code.to_sym]
-          else
-            nil
-          end
+          lang_code = get_resource(source_locale)[:languages].detect { |_, val| val.downcase == language.downcase }.first
+          get_resource(dest_locale)[:languages][lang_code] if lang_code
         rescue
           nil
         end

data/lib/twitter_cldr/shared.rb

@@ -5,12 +5,13 @@
 
 module TwitterCldr
   module Shared
-    autoload :Calendar,    'twitter_cldr/shared/calendar'
-    autoload :CodePoint,   'twitter_cldr/shared/code_point'
-    autoload :Currencies,  'twitter_cldr/shared/currencies'
-    autoload :Languages,   'twitter_cldr/shared/languages'
-    autoload :Numbers,     'twitter_cldr/shared/numbers'
-    autoload :PhoneCodes,  'twitter_cldr/shared/phone_codes'
-    autoload :PostalCodes, 'twitter_cldr/shared/postal_codes'
+    autoload :Calendar,      'twitter_cldr/shared/calendar'
+    autoload :CodePoint,     'twitter_cldr/shared/code_point'
+    autoload :Currencies,    'twitter_cldr/shared/currencies'
+    autoload :LanguageCodes, 'twitter_cldr/shared/language_codes'
+    autoload :Languages,     'twitter_cldr/shared/languages'
+    autoload :Numbers,       'twitter_cldr/shared/numbers'
+    autoload :PhoneCodes,    'twitter_cldr/shared/phone_codes'
+    autoload :PostalCodes,   'twitter_cldr/shared/postal_codes'
   end
 end

data/lib/twitter_cldr/tokenizers/base.rb

@@ -90,14 +90,8 @@ module TwitterCldr
         @placeholders = {}
       end
 
-      def traverse(path, haystack = @resource)
-        path.inject(haystack) do |current, segment|
-          if current.is_a?(Hash) && current.has_key?(segment)
-            current[segment]
-          else
-            return
-          end
-        end
+      def traverse(path, hash = @resource)
+        TwitterCldr::Utils.traverse_hash(hash, path)
       end
 
       # expands all path symbols

data/lib/twitter_cldr/utils.rb

@@ -41,6 +41,14 @@ module TwitterCldr
         end
       end
 
+      def traverse_hash(hash, path)
+        return if path.empty?
+
+        path.inject(hash) do |current, key|
+          current.is_a?(Hash) ? current[key] : return
+        end
+      end
+
     end
 
   end

data/lib/twitter_cldr/version.rb

@@ -4,5 +4,5 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 
 module TwitterCldr
-  VERSION = "1.7.0"
+  VERSION = "1.8.0"
 end

data/lib/twitter_cldr.rb

@@ -3,9 +3,6 @@
 # Copyright 2012 Twitter, Inc
 # http://www.apache.org/licenses/LICENSE-2.0
 
-$:.push(File.dirname(__FILE__))
-$:.push(File.dirname(File.dirname(__FILE__)))
-
 $KCODE = 'UTF-8' unless RUBY_VERSION >= '1.9.0'
 
 require 'yaml'
@@ -22,6 +19,7 @@ module TwitterCldr
 
   autoload :Formatters,    'twitter_cldr/formatters'
   autoload :Collation,     'twitter_cldr/collation'
+  autoload :Localized,     'twitter_cldr/localized'
   autoload :Normalization, 'twitter_cldr/normalization'
   autoload :Resources,     'twitter_cldr/resources'
   autoload :Shared,        'twitter_cldr/shared'
@@ -38,6 +36,8 @@ module TwitterCldr
 
   RESOURCES_DIR = File.join(File.dirname(File.dirname(File.expand_path(__FILE__))), 'resources')
 
+  JS_LIB_DIR = File.join(File.dirname(File.dirname(__FILE__)), 'js', 'lib')
+
   # maps twitter locales to cldr locales
   TWITTER_LOCALE_MAP = {
       :msa     => :ms,
@@ -88,7 +88,8 @@ module TwitterCldr
     end
 
     def require_js
-      require "js/lib/twitter_cldr_js"
+      $:.push(JS_LIB_DIR) unless $:.include?(JS_LIB_DIR)
+      require 'twitter_cldr_js'
     end
   end
 

data/resources/custom/locales/cs/units.yml

@@ -18,6 +18,6 @@
         :other: ! '{0}min'
     :second:
       :abbreviated:
-        :one: ! '{0}s'
-        :few: ! '{0}s'
-        :other: ! '{0}s'
+        :one: ! '{0}sec'
+        :few: ! '{0}sec'
+        :other: ! '{0}sec'

data/resources/custom/locales/pl/units.yml

@@ -15,10 +15,10 @@
         :other: ! '{0}godz.'
     :minute:
       :abbreviated:
-        :one: ! '{0}min'
-        :few: ! '{0}min'
-        :many: ! '{0}min'
-        :other: ! '{0}min'
+        :one: ! '{0} min'
+        :few: ! '{0} min'
+        :many: ! '{0} min'
+        :other: ! '{0} min'
     :second:
       :abbreviated:
         :one: ! '{0}s'

data/resources/custom/locales/pt/units.yml

@@ -15,5 +15,5 @@
         :other: ! '{0} m'
     :second:
       :abbreviated:
-        :one: ! '{0}s'
-        :other: ! '{0}s'
+        :one: ! '{0} s'
+        :other: ! '{0} s'

data/spec/core_ext_spec.rb

@@ -0,0 +1,19 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'spec_helper'
+
+describe 'Core classes localization' do
+
+  [Array, Bignum, Date, DateTime, Fixnum, Float, String, Symbol, Time].each do |klass|
+    describe klass do
+      it 'has public instance method #localize' do
+        # convert methods names to symbols (they're strings in 1.8)
+        klass.public_instance_methods.map(&:to_sym).should include(:localize)
+      end
+    end
+  end
+
+end

data/spec/{core_ext/array_spec.rb → localized/localized_array_spec.rb}

@@ -5,7 +5,7 @@
 
 require 'spec_helper'
 
-include TwitterCldr
+include TwitterCldr::Localized
 
 describe LocalizedArray do
   describe '#code_points_to_string' do

data/spec/{core_ext/calendars/date_spec.rb → localized/localized_date_spec.rb}

@@ -5,42 +5,13 @@
 
 require 'spec_helper'
 
-include TwitterCldr
+include TwitterCldr::Localized
 
-describe Date do
-  describe "#localize" do
-    before(:all) do
-      @date = Date.today
-    end
-    
-    it "should localize with the given locale, English by default" do
-      loc_date = @date.localize
-      loc_date.should be_a(LocalizedDate)
-      loc_date.locale.should == :en
-      loc_date.calendar_type.should == :gregorian
-      loc_date.base_obj.should == @date
-
-      loc_date = @date.localize(:it)
-      loc_date.should be_a(LocalizedDate)
-      loc_date.locale.should == :it
-    end
-
-    it "should localize with the given calendar" do
-      loc_date = @date.localize(:th, :calendar_type => :buddhist)
-      loc_date.should be_a(LocalizedDate)
-      loc_date.locale.should == :th
-      loc_date.calendar_type.should == :buddhist
-      loc_date.base_obj.should == @date
-    end
+describe LocalizedDate do
 
-    it "should forward calendar_type" do
-      loc_date = @date.localize(:th, :calendar_type => :buddhist)
-      loc_date.to_datetime(Time.now).calendar_type.should == :buddhist
-    end
+  let(:date) { Date.today }
 
-  end
-
-  describe "ago" do
+  describe "#ago" do
     let(:date) { Date.new(2010,7,6) }
     let(:base_time) { Time.gm(2010,8,6,12,12,30) }
 
@@ -91,7 +62,7 @@ describe Date do
     end
   end
 
-  describe "until" do
+  describe "#until" do
     let(:base_time) { Time.gm(2010,8,6,12,12,30) }
 
     it "should until-ify with a number of different units" do
@@ -114,10 +85,10 @@ describe Date do
 
   describe "stringify" do
     it "should stringify with a default calendar" do
-      #Date.today.localize(:th, :calendar_type => :buddhist).to_full_s # It doesn't support era
-      Date.today.localize(:th).to_long_s
-      Date.today.localize(:th).to_medium_s
-      Date.today.localize(:th).to_short_s
+      #date.localize(:th, :calendar_type => :buddhist).to_full_s # It doesn't support era
+      date.localize(:th).to_long_s
+      date.localize(:th).to_medium_s
+      date.localize(:th).to_short_s
     end
 
     it "should stringify with buddhist calendar" do
@@ -126,15 +97,13 @@ describe Date do
           be_nil, 'buddhist calendar is missing for :th locale (check resources/locales/th/calendars.yml)'
       )
 
-      #Date.today.localize(:th, :calendar_type => :buddhist).to_full_s # It doesn't support era
-      Date.today.localize(:th, :calendar_type => :buddhist).to_long_s
-      Date.today.localize(:th, :calendar_type => :buddhist).to_medium_s
-      Date.today.localize(:th, :calendar_type => :buddhist).to_short_s
+      #date.localize(:th, :calendar_type => :buddhist).to_full_s # It doesn't support era
+      date.localize(:th, :calendar_type => :buddhist).to_long_s
+      date.localize(:th, :calendar_type => :buddhist).to_medium_s
+      date.localize(:th, :calendar_type => :buddhist).to_short_s
     end
   end
-end
 
-describe LocalizedDate do
   describe "#to_datetime" do
     it "should combine a date and a time object into a datetime" do
       date = Date.new(1987, 9, 20)
@@ -152,4 +121,15 @@ describe LocalizedDate do
       datetime.base_obj.strftime("%Y-%m-%d %H:%M:%S").should == "1987-09-20 22:05:00"
     end
   end
+
+  describe 'formatters' do
+    it "don't raise errors for any locale" do
+      TwitterCldr.supported_locales.each do |locale|
+        TwitterCldr::Tokenizers::DateTimeTokenizer::VALID_TYPES.each do |type|
+          lambda { Date.today.localize(locale).send(:"to_#{type}_s") }.should_not raise_error
+        end
+      end
+    end
+  end
+
 end