twitter_cldr 1.6.2 → 1.7.0

data/lib/twitter_cldr/resources/download.rb

@@ -0,0 +1,41 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'cldr/download'
+
+module TwitterCldr
+  module Resources
+
+    CLDR_URL = 'http://unicode.org/Public/cldr/21/core.zip'
+
+    class << self
+
+      def download_if_necessary(path, url)
+        if File.file?(path)
+          puts "Using '#{path}'."
+        else
+          puts "Downloading '#{url}' to '#{path}'."
+          FileUtils.mkdir_p(File.dirname(path))
+          system("curl #{url} -o #{path}")
+        end
+
+        path
+      end
+
+      def download_cldr_if_necessary(path, url = CLDR_URL)
+        if File.directory?(path)
+          puts "Using CLDR data from '#{path}'."
+        else
+          puts "Downloading CLDR data from '#{url}' to '#{path}'."
+          Cldr.download(url, path)
+        end
+
+        path
+      end
+
+    end
+
+  end
+end

data/lib/twitter_cldr/resources/loader.rb

@@ -27,7 +27,7 @@ module TwitterCldr
       end
 
       def load_resource(path, merge_custom = true)
-        base = TwitterCldr::Utils.deep_symbolize_keys(YAML.load(read_resource_file(path)))
+        base = YAML.load(read_resource_file(path))
         custom_path = File.join("custom", path)
 
         if merge_custom && resource_exists?(custom_path)

data/lib/twitter_cldr/resources/locales_resources_importer.rb

@@ -0,0 +1,105 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'cldr/export'
+
+require 'lib/twitter_cldr/resources/download'
+
+module TwitterCldr
+  module Resources
+
+    class LocalesResourcesImporter
+
+      COMPONENTS = %w[calendars languages numbers units plurals]
+
+      # Arguments:
+      #
+      #   input_path  - path to a directory containing CLDR data
+      #   output_path - output directory for imported YAML files
+      #
+      def initialize(input_path, output_path)
+        @input_path  = input_path
+        @output_path = output_path
+      end
+
+      def import
+        prepare_ruby_cldr
+        import_components
+      end
+
+      private
+
+      def prepare_ruby_cldr
+        TwitterCldr::Resources.download_cldr_if_necessary(@input_path)
+        Cldr::Export::Data.dir = File.join(@input_path, 'common')
+      end
+
+      # Copies zh plurals to zh-Hant (they can share, but locale code needs to be different).
+      #
+      def copy_zh_hant_plurals
+        File.open(File.join(@output_path, 'zh-Hant', 'plurals.yml'), 'w:utf-8') do |output|
+          data = YAML.load(File.read(File.join(@output_path, 'zh', 'plurals.yml')))
+          output.write(YAML.dump(:'zh-Hant' => data[:zh].gsub(':zh', ":'zh-Hant'")))
+        end
+      end
+
+      def import_components
+        Cldr::Export.export(:locales => TwitterCldr.supported_locales, :components => COMPONENTS, :target => @output_path) do |component, locale, path|
+          add_buddhist_calendar(component, locale, path)
+          process_plurals(component, locale, path)
+          deep_symbolize(component, locale, path)
+        end
+
+        copy_zh_hant_plurals
+      end
+
+      def deep_symbolize(component, locale, path)
+        return unless File.extname(path) == '.yml'
+        data = YAML.load(File.read(path))
+
+        File.open(path, 'w:utf-8') do |output|
+          output.write(YAML.dump(TwitterCldr::Utils.deep_symbolize_keys(data)))
+        end
+      end
+
+      def process_plurals(component, locale, path)
+        return unless component == 'plurals'
+
+        plural_rules = File.read(path)
+
+        File.open(path.gsub(/rb$/, 'yml'), 'w:utf-8') do |output|
+          output.write(YAML.dump({ locale => plural_rules }))
+        end
+
+        FileUtils.rm(path)
+      end
+
+      # TODO: export buddhist calendar from CLDR data instead of using BUDDHIST_CALENDAR constant.
+      #
+      def add_buddhist_calendar(component, locale, path)
+        return unless component == 'calendars' && locale == :th
+
+        data = YAML.load(File.read(path))
+        data['th']['calendars']['buddhist'] = BUDDHIST_CALENDAR
+
+        File.open(path, 'w:utf-8') { |output| output.write(YAML.dump(data))}
+      end
+
+      BUDDHIST_CALENDAR = {
+          'formats' => {
+              'date' => {
+                  'default' => :'calendars.buddhist.formats.date.medium',
+                  'full'    => { 'pattern' => 'EEEEที่ d MMMM G y' },
+                  'long'    => { 'pattern' => 'd MMMM พ.ศ. #{y + 543}' },
+                  'medium'  => { 'pattern' => 'd MMM #{y + 543}' },
+                  'short'   => { 'pattern' => 'd/M/#{y + 543}' }
+              }
+          }
+      }
+
+    end
+
+  end
+end

data/lib/twitter_cldr/resources/phone_codes_importer.rb

@@ -0,0 +1,48 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'nokogiri'
+
+require 'lib/twitter_cldr/resources/download'
+
+module TwitterCldr
+  module Resources
+
+    class PhoneCodesImporter
+
+      PHONE_CODES_PATH = File.join('common', 'supplemental', 'telephoneCodeData.xml')
+
+      TERRITORY_REGEXP = /^[A-Z]{2}$/
+
+      # Arguments:
+      #
+      #   input_path  - path to a directory containing CLDR data
+      #   output_path - output directory for generated YAML file
+      #
+      def initialize(input_path, output_path)
+        @input_path  = input_path
+        @output_path = output_path
+      end
+
+      def import
+        TwitterCldr::Resources.download_cldr_if_necessary(@input_path)
+
+        doc = File.open(File.join(@input_path, PHONE_CODES_PATH)) { |file| Nokogiri::XML(file) }
+
+        phone_codes = doc.xpath('//codesByTerritory').inject({}) do |memo, node|
+          territory = node.attr('territory')
+          memo[territory.downcase.to_sym] = node.at_xpath('telephoneCountryCode').attr('code') if territory =~ TERRITORY_REGEXP
+          memo
+        end
+
+        phone_codes = Hash[phone_codes.sort_by(&:first)]
+
+        File.open(File.join(@output_path, 'phone_codes.yml'), 'w') { |output| output.write(YAML.dump(phone_codes)) }
+      end
+
+    end
+
+  end
+end

data/lib/twitter_cldr/resources/postal_codes_importer.rb

@@ -0,0 +1,44 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'nokogiri'
+
+require 'lib/twitter_cldr/resources/download'
+
+module TwitterCldr
+  module Resources
+
+    class PostalCodesImporter
+
+      POSTAL_CODES_PATH = File.join('common', 'supplemental', 'postalCodeData.xml')
+
+      # Arguments:
+      #
+      #   input_path  - path to a directory containing CLDR data
+      #   output_path - output directory for generated YAML file
+      #
+      def initialize(input_path, output_path)
+        @input_path  = input_path
+        @output_path = output_path
+      end
+
+      def import
+        TwitterCldr::Resources.download_cldr_if_necessary(@input_path)
+
+        doc = File.open(File.join(@input_path, POSTAL_CODES_PATH)) { |file| Nokogiri::XML(file) }
+
+        postal_codes = doc.xpath('//postCodeRegex').inject({}) do |memo, node|
+          memo[node.attr('territoryId').downcase.to_sym] = Regexp.new(node.text); memo
+        end
+
+        postal_codes = Hash[postal_codes.sort_by(&:first)]
+
+        File.open(File.join(@output_path, 'postal_codes.yml'), 'w') { |output| output.write(YAML.dump(postal_codes)) }
+      end
+
+    end
+
+  end
+end

data/lib/twitter_cldr/resources/tailoring_importer.rb

@@ -4,9 +4,10 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 
 require 'nokogiri'
-require 'yaml'
 require 'java'
 
+require 'lib/twitter_cldr/resources/download'
+
 module TwitterCldr
   module Resources
     # This class should be used with JRuby 1.7 in 1.9 mode and ICU4J version 49.1 (available at
@@ -31,18 +32,14 @@ module TwitterCldr
           :pt => :root
       }
 
-      EMPTY_TAILORING_DATA = { 'collator_options' => {}, 'tailored_table' => '', 'suppressed_contractions' => '' }
+      EMPTY_TAILORING_DATA = { :collator_options => {}, :tailored_table => '', :suppressed_contractions => '' }
 
       class ImportError < RuntimeError; end
 
       # Arguments:
       #
-      #   input_path  - path to a directory containing CLDR tailoring data (available at
-      #                 http://unicode.org/cldr/trac/browser/tags/release-21/common/collation/
-      #                 or as a part of CLDR release at http://cldr.unicode.org/index/downloads)
-      #
+      #   input_path  - path to a directory containing CLDR data
       #   output_path - output directory for imported YAML files
-      #
       #   icu4j_path  - path to ICU4J jar file
       #
       def initialize(input_path, output_path, icu4j_path)
@@ -52,7 +49,14 @@ module TwitterCldr
         @output_path = output_path
       end
 
-      def import(locale)
+      def import(locales)
+        TwitterCldr::Resources.download_cldr_if_necessary(@input_path)
+        locales.each { |locale| import_locale(locale) }
+      end
+
+      private
+
+      def import_locale(locale)
         print "Importing %8s\t--\t" % locale
 
         if tailoring_present?(locale)
@@ -66,8 +70,6 @@ module TwitterCldr
         puts "Error: #{e.message}"
       end
 
-      private
-
       def dump(locale, data)
         File.open(resource_file_path(locale), 'w') { |file| YAML.dump(data, file) }
       end
@@ -81,7 +83,7 @@ module TwitterCldr
       end
 
       def locale_file_path(locale)
-        File.join(@input_path, "#{translated_locale(locale)}.xml")
+        File.join(@input_path, 'common', 'collation', "#{translated_locale(locale)}.xml")
       end
 
       def resource_file_path(locale)
@@ -100,9 +102,9 @@ module TwitterCldr
         standard_tailoring = collations.at_xpath('collation[@type="standard"]')
 
         {
-            'collator_options'        => parse_collator_options(standard_tailoring),
-            'tailored_table'          => parse_tailorings(standard_tailoring, locale),
-            'suppressed_contractions' => parse_suppressed_contractions(standard_tailoring)
+            :collator_options        => parse_collator_options(standard_tailoring),
+            :tailored_table          => parse_tailorings(standard_tailoring, locale),
+            :suppressed_contractions => parse_suppressed_contractions(standard_tailoring)
         }
       end
 
@@ -140,7 +142,7 @@ module TwitterCldr
       end
 
       def table_entry_for_rule(collator, tailored_value)
-        code_points = get_code_points(tailored_value)
+        code_points = get_code_points(tailored_value).map { |cp| cp.to_s(16).upcase.rjust(4, '0') }
 
         collation_elements = get_collation_elements(collator, tailored_value).map do |ce|
           ce.map { |l| l.to_s(16).upcase }.join(', ')
@@ -162,7 +164,7 @@ module TwitterCldr
           options['case_first'] = case_first_setting.attr('caseFirst').to_sym if case_first_setting
         end
 
-        options
+        TwitterCldr::Utils.deep_symbolize_keys(options)
       end
 
       def validate_tailoring_rule(rule)

data/lib/twitter_cldr/resources/unicode_data_importer.rb

@@ -0,0 +1,90 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'lib/twitter_cldr/resources/download'
+
+module TwitterCldr
+  module Resources
+
+    class UnicodeDataImporter
+
+      BLOCKS_URL       = 'ftp://ftp.unicode.org/Public/UNIDATA/Blocks.txt'
+      UNICODE_DATA_URL = 'ftp://ftp.unicode.org/Public/UNIDATA/UnicodeData.txt'
+
+      # Arguments:
+      #
+      #   input_path  - path to a directory containing Blocks.txt and UnicodeData.txt
+      #   output_path - output directory for imported YAML files
+      #
+      def initialize(input_path, output_path)
+        @input_path  = input_path
+        @output_path = output_path
+      end
+
+      def import
+        blocks       = import_blocks
+        unicode_data = import_unicode_data(blocks)
+
+        File.open(File.join(@output_path, 'blocks.yml'), 'w') { |output| YAML.dump(blocks, output) }
+
+        unicode_data.each do |block_name, code_points|
+          File.open(File.join(@output_path, 'blocks', "#{block_name}.yml"), 'w') { |output| YAML.dump(code_points, output) }
+        end
+      end
+
+      private
+
+      def import_blocks
+        blocks = {}
+
+        File.open(blocks_file) do |input|
+          input.each_line do |line|
+            next unless line =~ /^([0-9A-F]+)\.\.([0-9A-F]+);(.+)$/
+
+            range = ($1.hex..$2.hex)
+            name  = block_name($3)
+
+            blocks[name.to_sym] = range
+          end
+        end
+
+        blocks
+      end
+
+      def import_unicode_data(blocks)
+        unicode_data = Hash.new { |hash, key| hash[key] = Hash.new { |h, k| h[k] = {} } }
+
+        File.open(unicode_data_file) do |input|
+          input.each_line do |line|
+            data = line.chomp.split(';', -1)
+            data[0] = data[0].hex
+
+            unicode_data[find_block(blocks, data[0]).first][data[0]] = data
+          end
+        end
+
+        unicode_data
+      end
+
+      def unicode_data_file
+        TwitterCldr::Resources.download_if_necessary(File.join(@input_path, 'UnicodeData.txt'), UNICODE_DATA_URL)
+      end
+
+      def blocks_file
+        TwitterCldr::Resources.download_if_necessary(File.join(@input_path, 'Blocks.txt'), BLOCKS_URL)
+      end
+
+      def find_block(blocks, code_point)
+        blocks.detect { |_, range| range.include?(code_point) }
+      end
+
+      def block_name(string)
+        string.strip.downcase.gsub(/[\s-]/, '_')
+      end
+
+    end
+
+  end
+end

data/lib/twitter_cldr/shared.rb

@@ -6,9 +6,11 @@
 module TwitterCldr
   module Shared
     autoload :Calendar,    'twitter_cldr/shared/calendar'
+    autoload :CodePoint,   'twitter_cldr/shared/code_point'
     autoload :Currencies,  'twitter_cldr/shared/currencies'
     autoload :Languages,   'twitter_cldr/shared/languages'
     autoload :Numbers,     'twitter_cldr/shared/numbers'
-    autoload :CodePoint,   'twitter_cldr/shared/code_point'
+    autoload :PhoneCodes,  'twitter_cldr/shared/phone_codes'
+    autoload :PostalCodes, 'twitter_cldr/shared/postal_codes'
   end
 end