twitter_cldr 3.0.0.beta1 → 3.0.0

data/lib/twitter_cldr/resources/rbnf_test_importer.rb

@@ -0,0 +1,107 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'java'
+require 'fileutils'
+require 'twitter_cldr/resources/download'
+
+module TwitterCldr
+  module Resources
+
+    # This class should be used with JRuby in 1.9 mode
+    class RbnfTestImporter < IcuBasedImporter
+
+      attr_reader :output_path, :icu4j_path
+
+      # These don't have much of a pattern, just trying to get
+      # a wide range of different possibilities.
+      TEST_NUMBERS = [
+        [-1_141, -1_142, -1_143],
+        [-100, -75, -50, -24],
+        (0..100),
+        [321, 322, 323, 1_141, 1_142, 1_143, 10_311, 138_400]
+        # [41.0, 5.22, 8.90, 555.1212, -14.90, -999.701]  # decimals really aren't supported yet
+      ]
+
+      def initialize(output_path, icu4j_path)
+        @output_path = output_path
+        @icu4j_path = icu4j_path
+      end
+
+      def import(locales)
+        require_icu4j(icu4j_path)
+        java_import 'com.ibm.icu.text.RuleBasedNumberFormat'
+        java_import 'com.ibm.icu.util.ULocale'
+
+        groupings = [
+          RuleBasedNumberFormat::SPELLOUT,
+          RuleBasedNumberFormat::ORDINAL,
+          RuleBasedNumberFormat::DURATION
+        ]
+
+        import_locales(locales, groupings)
+      end
+
+      protected
+
+      def import_locales(locales, groupings)
+        locales.each do |locale|
+          locale = locale.to_s
+          ulocale = ULocale.new(locale)
+          file = output_file_for(locale)
+          FileUtils.mkdir_p(File.dirname(file))
+          File.open(file, "w+") do |w|
+            w.write(YAML.dump(import_locale(ulocale, groupings)))
+          end
+        end
+      end
+
+      def import_locale(ulocale, groupings)
+        groupings.inject({}) do |grouping_ret, grouping|
+          formatter = RuleBasedNumberFormat.new(ulocale, grouping)
+          grouping_name = get_grouping_display_name(grouping)
+          grouping_ret[grouping_name] = formatter.getRuleSetNames.inject({}) do |ruleset_ret, ruleset_name|
+            ruleset_display_name = formatter.getRuleSetDisplayName(ruleset_name, ulocale)
+            ruleset_display_name = clean_up_name(ruleset_display_name)
+            ruleset_ret[ruleset_display_name] = import_ruleset(formatter, ruleset_name)
+            ruleset_ret
+          end
+          grouping_ret
+        end
+      end
+
+      def clean_up_name(name)
+        name
+          .gsub(/[^\w-]/, '-')
+          .gsub('GREEKNUMERALMAJUSCULES', 'GreekNumeralMajuscules')
+      end
+
+      def import_ruleset(formatter, ruleset_name)
+        TEST_NUMBERS.inject({}) do |ret, num_set|
+          num_set.each do |num|
+            ret[num] = formatter.format(num, ruleset_name)
+          end
+          ret
+        end
+      end
+
+      def output_file_for(locale)
+        File.join(output_path, locale, "rbnf_test.yml")
+      end
+
+      def get_grouping_display_name(grouping)
+        case grouping
+          when RuleBasedNumberFormat::SPELLOUT
+            "SpelloutRules"
+          when RuleBasedNumberFormat::ORDINAL
+            "OrdinalRules"
+          when RuleBasedNumberFormat::DURATION
+            "DurationRules"
+        end
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/resources/readme_renderer.rb

@@ -0,0 +1,115 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'erb'
+
+# patch to add compare operator to Symbol (for mri 1.8)
+class Symbol
+  def <=>(other)
+    to_s <=> other.to_s
+  end
+end
+
+module TwitterCldr
+  module Resources
+
+    ReadmeAssertionFailure = Struct.new(:message, :line_number)
+
+    class ReadmeRenderer
+
+      attr_reader :text, :assertion_failures
+
+      def initialize(text)
+        @text = text
+        @assertion_failures = []
+      end
+
+      def render
+        ERB.new(text).result(binding)
+      end
+
+      def datetime
+        @datetime ||= DateTime.new(2014, 2, 14, 12, 20, 5, 0)
+      end
+
+      def time
+        @time ||= Time.gm(2014, 2, 14, 12, 20, 5, 0)
+      end
+
+      private
+
+      def assert(got, expected)
+        if got.is_a?(String) && expected.is_a?(String)
+          got = got.localize.normalize(:using => :NFKC).to_s
+          expected = expected.localize.normalize(:using => :NFKC).to_s
+        end
+
+        unless equal?(got, expected)
+          line_num = line_num_from_stack_trace(Kernel.caller)
+          assertion_failures << ReadmeAssertionFailure.new(
+            "Expected `#{got.inspect}` to be `#{expected.inspect}` in README on line #{line_num}",
+            line_num
+          )
+        end
+
+        got
+      end
+
+      def equal?(obj1, obj2)
+        case obj1
+          when Array
+            obj1 - obj2 == []
+          else
+            obj1 == obj2
+        end
+      end
+
+      def assert_true(got)
+        assert(got, true)
+      end
+
+      def assert_false(got)
+        assert(got, false)
+      end
+
+      def assert_no_error(proc)
+        error = nil
+        begin
+          proc.call
+        rescue => e
+          line_num = line_num_from_stack_trace(Kernel.caller)
+          assertion_failures << ReadmeAssertionFailure.new(
+            "Expected README line #{line_num} not to raise an exception, but it did:\n#{e.message}\n#{e.backtrace.join("\n")}",
+            line_num
+          )
+        end
+      end
+
+      private
+
+      def line_num_from_stack_trace(trace)
+        trace[0].split(":")[1].to_i  # kind of a hack...
+      end
+
+      def ellipsize(obj)
+        case obj
+          when Array
+            "[#{obj.map(&:inspect).join(", ")}, ... ]"
+          when Hash
+            hash_text = obj.map { |key, val| "#{key.inspect} => #{val.inspect}" }.join(", ")
+            "{ ... #{hash_text} ... }"
+        end
+      end
+
+      def slice_hash(hash, keys)
+        hash.inject({}) do |ret, (key, val)|
+          ret[key] = val if keys.include?(key)
+          ret
+        end
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/resources/tailoring_importer.rb

@@ -10,10 +10,9 @@ require 'twitter_cldr/resources/download'
 
 module TwitterCldr
   module Resources
-    # This class should be used with JRuby 1.7 in 1.9 mode and ICU4J version 49.1 (available at
-    # http://download.icu-project.org/files/icu4j/49.1/icu4j-49_1.jar).
+    # This class should be used with JRuby 1.7 in 1.9 mode and ICU4J version >= 49.1.
     #
-    class TailoringImporter
+    class TailoringImporter < IcuBasedImporter
 
       SUPPORTED_RULES   = %w[p s t i pc sc tc ic x]
       SIMPLE_RULES      = %w[p s t i]
@@ -60,11 +59,6 @@ module TwitterCldr
 
       private
 
-      def require_icu4j(icu4j_path)
-        TwitterCldr::Resources.download_icu4j_if_necessary(icu4j_path)
-        require icu4j_path
-      end
-
       def import_locale(locale)
         print "Importing %8s\t--\t" % locale
 

data/lib/twitter_cldr/resources/uli/segment_exceptions_importer.rb

@@ -0,0 +1,62 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'twitter_cldr/resources/download'
+require 'fileutils'
+require 'json'
+
+module TwitterCldr
+  module Resources
+    module Uli
+      class SegmentExceptionsImporter
+
+        URL = "http://unicode.org/uli/trac/export/38/trunk/abbrs/json/%{locale}.json"
+
+        attr_reader :input_path, :output_path
+
+        def initialize(input_path, output_path)
+          @input_path  = input_path
+          @output_path = output_path
+        end
+
+        def import(locales)
+          FileUtils.mkdir_p(input_path)
+          FileUtils.mkdir_p(output_path)
+          locales.each { |locale| import_locale(locale) }
+        end
+
+        private
+
+        def import_locale(locale)
+          if input_file = download_resource_for(locale)
+            output_file = File.join(output_path, "#{locale}.yml")
+            exceptions = JSON.parse(File.read(input_file))
+
+            File.open(output_file, "w+") do |f|
+              YAML.dump({
+                locale => {
+                  :exceptions => exceptions["data"]["abbrs"]
+                }
+              }, f)
+            end
+          end
+        end
+
+        def download_resource_for(locale)
+          input_file = input_path_for(locale)
+          TwitterCldr::Resources.download_if_necessary(
+            input_file, URL.gsub("%{locale}", locale.to_s)
+          )
+          input_file
+        end
+
+        def input_path_for(locale)
+          File.join(input_path, "#{locale}.json")
+        end
+
+      end
+    end
+  end
+end

data/lib/twitter_cldr/resources/uli.rb

@@ -0,0 +1,12 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Resources
+    module Uli
+      autoload :SegmentExceptionsImporter, 'twitter_cldr/resources/uli/segment_exceptions_importer'
+    end
+  end
+end

data/lib/twitter_cldr/resources/unicode_data_importer.rb

@@ -4,14 +4,16 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 
 require 'twitter_cldr/resources/download'
+require 'pry-nav'
 
 module TwitterCldr
   module Resources
 
-    class UnicodeDataImporter
+    class UnicodeDataImporter < UnicodeImporter
 
-      BLOCKS_URL       = 'ftp://ftp.unicode.org/Public/UNIDATA/Blocks.txt'
-      UNICODE_DATA_URL = 'ftp://ftp.unicode.org/Public/UNIDATA/UnicodeData.txt'
+      BLOCKS_URL           = 'ftp://ftp.unicode.org/Public/UNIDATA/Blocks.txt'
+      UNICODE_DATA_URL     = 'ftp://ftp.unicode.org/Public/UNIDATA/UnicodeData.txt'
+      CASEFOLDING_DATA_URL = 'ftp://ftp.unicode.org/Public/UNIDATA/CaseFolding.txt'
 
       # Arguments:
       #
@@ -24,15 +26,49 @@ module TwitterCldr
       end
 
       def import
-        blocks       = import_blocks
-        unicode_data = import_unicode_data(blocks)
-
-        File.open(File.join(@output_path, 'blocks.yml'), 'w') { |output| YAML.dump(blocks, output) }
+        blocks           = import_blocks
+        unicode_data     = import_unicode_data(blocks)
+        casefolding_data = import_casefolding_data
+        index_data       = build_indices(
+          TwitterCldr::Shared::CodePoint::INDICES, unicode_data
+        )
+
+        File.open(File.join(@output_path, 'blocks.yml'), 'w') do |output|
+          YAML.dump(blocks, output)
+        end
 
         FileUtils.mkdir_p(File.join(@output_path, 'blocks'))
 
         unicode_data.each do |block_name, code_points|
-          File.open(File.join(@output_path, 'blocks', "#{block_name}.yml"), 'w') { |output| YAML.dump(code_points, output) }
+          File.open(File.join(@output_path, 'blocks', "#{block_name}.yml"), 'w') do |output|
+            YAML.dump(code_points, output)
+          end
+        end
+
+        File.open(File.join(@output_path, 'casefolding.yml'), 'w') do |output|
+          YAML.dump(casefolding_data, output)
+        end
+
+        FileUtils.mkdir_p(File.join(@output_path, 'indices'))
+
+        indices = index_data
+
+        indices.each_pair do |index_name, data|
+          File.open(File.join(@output_path, "indices", "#{index_name}.yml"), 'w') do |output|
+            YAML.dump(data, output)
+          end
+        end
+
+        File.open(File.join(@output_path, "indices", "keys.yml"), 'w') do |output|
+          YAML.dump(
+            indices.inject({}) do |ret, (index_name, data)|
+              data.keys.each do |prop|
+                ret[prop] ||= []
+                ret[prop] << index_name
+              end
+              ret
+            end, output
+          )
         end
       end
 
@@ -58,16 +94,50 @@ module TwitterCldr
       def import_unicode_data(blocks)
         unicode_data = Hash.new { |hash, key| hash[key] = Hash.new { |h, k| h[k] = {} } }
 
-        File.open(unicode_data_file) do |input|
-          input.each_line do |line|
-            data = line.chomp.split(';', -1)
-            data[0] = data[0].hex
+        parse_standard_file(unicode_data_file) do |data|
+          data[0] = data[0].hex
+          unicode_data[find_block(blocks, data[0]).first][data[0]] = data
+        end
 
-            unicode_data[find_block(blocks, data[0]).first][data[0]] = data
+        unicode_data
+      end
+
+      def import_casefolding_data
+        parse_standard_file(casefold_data_file).map do |data|
+          {
+            :source => data[0].hex,
+            :target => data[2].split(" ").map(&:hex),
+            :status => data[1]
+          }
+        end
+      end
+
+      def build_indices(indices, unicode_data)
+        indices.inject({}) do |index_ret, index_name|
+          field_index = TwitterCldr::Shared::CODE_POINT_FIELDS.find_index do |field|
+            field == index_name
           end
+
+          index_ret[index_name] = Hash.new { |hash, key| hash[key] = [] }
+
+          unicode_data.each_pair do |block_name, block_data|
+            block_data.each_pair do |code_point, data|
+              index_ret[index_name][data[field_index].to_sym] << code_point
+            end
+          end
+
+          index_ret
+        end.inject({}) do |index_ret, (index_key, index_data)|
+          index_ret[index_key] = index_data.inject({}) do |field_ret, (field_key, field_data)|
+            field_ret[field_key] = TwitterCldr::Utils::RangeSet.rangify(field_data)
+            field_ret
+          end
+          index_ret
         end
+      end
 
-        unicode_data
+      def casefold_data_file
+        TwitterCldr::Resources.download_if_necessary(File.join(@input_path, 'CaseFolding.txt'), CASEFOLDING_DATA_URL)
       end
 
       def unicode_data_file

data/lib/twitter_cldr/resources/unicode_importer.rb

@@ -0,0 +1,37 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Resources
+    class UnicodeImporter
+
+      protected
+
+      def parse_standard_file(file)
+        if block_given?
+          File.open(file) do |input|
+            input.each_line do |line|
+              unless line.split[0] == "#"
+                comment_idx = if idx = line.index("#")
+                  idx - 1 # consume #
+                else
+                  line.size
+                end
+
+                line = line.chomp[0..comment_idx]
+                if line.size > 0
+                  yield line.split(';', -1).map(&:strip)
+                end
+              end
+            end
+          end
+        else
+          enum_for(__method__, file)
+        end
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/resources/unicode_properties_importer.rb

@@ -0,0 +1,79 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'twitter_cldr/resources/download'
+
+module TwitterCldr
+  module Resources
+
+    class UnicodePropertiesImporter < UnicodeImporter
+
+      PROPERTIES_BASE_URL = 'ftp://ftp.unicode.org/Public/UCD/latest/ucd'
+      PROPERTIES = [
+        "auxiliary/SentenceBreakProperty",
+        "auxiliary/WordBreakProperty",
+        "LineBreak"
+      ]
+
+      # Arguments:
+      #
+      #   input_path  - path to a directory containing the various property files
+      #   output_path - output directory for imported YAML files
+      #
+      def initialize(input_path, output_path)
+        @input_path  = input_path
+        @output_path = output_path
+      end
+
+      def import
+        FileUtils.mkdir_p(@output_path)
+
+        PROPERTIES.each do |property|
+          input_file = property_data_file("#{property}.txt")
+          output_file = File.join(@output_path, "#{fix_name(property)}.yml")
+
+          File.open(output_file, "w+") do |f|
+            f.write(
+              YAML.dump(
+                parse_standard_file(input_file).inject({}) do |ret, data|
+                  name = data[1].strip.to_sym
+                  ret[name] ||= []
+                  ret[name] += expand_range(data[0])
+                  ret
+                end.inject({}) do |ret, (key, data)|
+                  ret[key] = TwitterCldr::Utils::RangeSet.rangify(data)
+                  ret
+                end
+              )
+            )
+          end
+        end
+      end
+
+      private
+
+      def fix_name(str)
+        underscore(File.basename(str.gsub("Property", "")))
+      end
+
+      def underscore(str)
+        str.gsub(/([a-z])([A-Z])/, '\1_\2').downcase
+      end
+
+      def expand_range(str)
+        initial, final = str.split("..")
+        (initial.to_i(16)..(final || initial).to_i(16)).to_a
+      end
+
+      def property_data_file(file)
+        TwitterCldr::Resources.download_if_necessary(
+          File.join(@input_path, File.basename(file)), File.join(PROPERTIES_BASE_URL, file)
+        )
+      end
+
+    end
+
+  end
+end

data/lib/twitter_cldr/resources.rb

@@ -5,6 +5,9 @@
 
 module TwitterCldr
   module Resources
+    autoload :Uli,                                'twitter_cldr/resources/uli'
+    autoload :UnicodeImporter,                    'twitter_cldr/resources/unicode_importer'
+    autoload :IcuBasedImporter,                   'twitter_cldr/resources/icu_based_importer'
     autoload :CanonicalCompositionsUpdater,       'twitter_cldr/resources/canonical_compositions_updater'
     autoload :CollationTriesDumper,               'twitter_cldr/resources/collation_tries_dumper'
     autoload :CompositionExclusionsImporter,      'twitter_cldr/resources/composition_exclusions_importer'
@@ -17,7 +20,11 @@ module TwitterCldr
     autoload :PostalCodesImporter,                'twitter_cldr/resources/postal_codes_importer'
     autoload :TailoringImporter,                  'twitter_cldr/resources/tailoring_importer'
     autoload :UnicodeDataImporter,                'twitter_cldr/resources/unicode_data_importer'
+    autoload :UnicodePropertiesImporter,          'twitter_cldr/resources/unicode_properties_importer'
     autoload :BidiTestImporter,                   'twitter_cldr/resources/bidi_test_importer'
     autoload :NormalizationQuickCheckImporter,    'twitter_cldr/resources/normalization_quick_check_importer'
+    autoload :RbnfTestImporter,                   'twitter_cldr/resources/rbnf_test_importer'
+    autoload :ReadmeRenderer,                     'twitter_cldr/resources/readme_renderer'
+    autoload :CasefolderClassGenerator,           'twitter_cldr/resources/casefolder_class_generator'
   end
-end
+end