twitter_cldr 4.0.0 → 4.1.0

data/lib/twitter_cldr/resources/segment_tests_importer.rb

@@ -3,29 +3,22 @@
 # Copyright 2012 Twitter, Inc
 # http://www.apache.org/licenses/LICENSE-2.0
 
-require 'twitter_cldr/resources/download'
 require 'fileutils'
 
 module TwitterCldr
   module Resources
-    class SegmentTestsImporter < UnicodeImporter
+    class SegmentTestsImporter < Importer
 
-      URL_ROOT = "ucd/auxiliary"
       TEST_FILES = [
-        'WordBreakTest.txt', 'SentenceBreakTest.txt'
+        'ucd/auxiliary/WordBreakTest.txt',
+        'ucd/auxiliary/SentenceBreakTest.txt'
       ]
 
-      attr_reader :input_path, :output_path
-
-      def initialize(input_path, output_path)
-        @input_path  = input_path
-        @output_path = output_path
-      end
-
-      def import
-        FileUtils.mkdir_p(input_path)
-        FileUtils.mkdir_p(output_path)
+      requirement :unicode, Versions.unicode_version, TEST_FILES
+      output_path 'shared/segments/tests'
+      ruby_engine :mri
 
+      def execute
         TEST_FILES.each do |test_file|
           import_test_file(test_file)
         end
@@ -34,27 +27,19 @@ module TwitterCldr
       private
 
       def import_test_file(test_file)
-        url = "#{URL_ROOT}/#{test_file}"
-        input_file = input_file_for(test_file)
-        output_file = output_path_for(test_file)
-        download(input_file, url)
-        result = parse_standard_file(input_file).map(&:first)
-        File.write(output_file, YAML.dump(result))
+        source_file = source_path_for(test_file)
+        FileUtils.mkdir_p(File.dirname(source_file))
+        result = UnicodeFileParser.parse_standard_file(source_file).map(&:first)
+        File.write(output_path_for(test_file), YAML.dump(result))
       end
 
-      def input_file_for(test_file)
-        File.join(input_path, test_file)
+      def source_path_for(test_file)
+        requirements[:unicode].source_path_for(test_file)
       end
 
       def output_path_for(test_file)
-        base = underscore(test_file.chomp(File.extname(test_file)))
-        File.join(output_path, "#{base}.yml")
-      end
-
-      def download(input_file, url)
-        TwitterCldr::Resources.download_unicode_data_if_necessary(
-          input_file, url
-        )
+        file = underscore(File.basename(test_file).chomp(File.extname(test_file)))
+        File.join(params.fetch(:output_path), "#{file}.yml")
       end
 
       def underscore(str)

data/lib/twitter_cldr/resources/tailoring_importer.rb

@@ -4,16 +4,19 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 
 require 'nokogiri'
-require 'java'
-
-require 'twitter_cldr/resources/download'
 
 module TwitterCldr
   module Resources
     # This class should be used with JRuby 1.7 in 1.9 mode, ICU4J version >= 49.1,
     # and CLDR version <= 23 (v24 syntax is not supported yet).
     #
-    class TailoringImporter < IcuBasedImporter
+    class TailoringImporter < Importer
+
+      requirement :icu, '51.2'
+      requirement :cldr, '23.1'
+      output_path 'collation/tailoring'
+      locales TwitterCldr.supported_locales
+      ruby_engine :jruby
 
       SUPPORTED_RULES   = %w[p s t i pc sc tc ic x]
       SIMPLE_RULES      = %w[p s t i]
@@ -40,26 +43,12 @@ module TwitterCldr
 
       class ImportError < RuntimeError; end
 
-      # Arguments:
-      #
-      #   input_path  - path to a directory containing CLDR data
-      #   output_path - output directory for imported YAML files
-      #   icu4j_path  - path to ICU4J jar file
-      #
-      def initialize(input_path, output_path, icu4j_path)
-        require_icu4j(icu4j_path)
-
-        @input_path  = input_path
-        @output_path = output_path
-      end
+      private
 
-      def import(locales)
-        TwitterCldr::Resources.download_cldr_if_necessary(@input_path)
-        locales.each { |locale| import_locale(locale) }
+      def execute
+        params[:locales].each { |locale| import_locale(locale) }
       end
 
-      private
-
       def import_locale(locale)
         print "Importing %8s\t--\t" % locale
 
@@ -87,11 +76,13 @@ module TwitterCldr
       end
 
       def locale_file_path(locale)
-        File.join(@input_path, 'common', 'collation', "#{translated_locale(locale)}.xml")
+        File.join(
+          requirements[:cldr].common_path, 'collation', "#{translated_locale(locale)}.xml"
+        )
       end
 
       def resource_file_path(locale)
-        File.join(@output_path, "#{locale}.yml")
+        File.join(params[:output_path], "#{locale}.yml")
       end
 
       def tailoring_data(locale)
@@ -136,12 +127,28 @@ module TwitterCldr
         default_type_node && default_type_node.attr('type')
       end
 
+      def get_class(name)
+        requirements[:icu].get_class(name)
+      end
+
+      def collator_class
+        @collator_class ||= get_class('com.ibm.icu.text.Collator')
+      end
+
+      def unicode_set_class
+        @unicode_set_class ||= get_class('com.ibm.icu.text.UnicodeSet')
+      end
+
+      def collation_element_iterator_class
+        @collation_element_iterator_class ||= get_class('com.ibm.icu.text.CollationElementIterator')
+      end
+
       def parse_tailorings(data, locale)
         rules = data && data.at_xpath('rules')
 
         return '' unless rules
 
-        collator = Java::ComIbmIcuText::Collator.get_instance(Java::JavaUtil::Locale.new(locale.to_s))
+        collator = collator_class.get_instance(Java::JavaUtil::Locale.new(locale.to_s))
 
         rules.children.map do |child|
           validate_tailoring_rule(child)
@@ -183,7 +190,7 @@ module TwitterCldr
 
       def parse_suppressed_contractions(data)
         node = data && data.at_xpath('suppress_contractions')
-        node ? Java::ComIbmIcuText::UnicodeSet.to_array(Java::ComIbmIcuText::UnicodeSet.new(node.text)).to_a.join : ''
+        node ? unicode_set_class.to_array(unicode_set_class.new(node.text)).to_a.join : ''
       end
 
       def parse_collator_options(data)
@@ -209,7 +216,7 @@ module TwitterCldr
         collation_elements = []
         ce = iter.next
 
-        while ce != Java::ComIbmIcuText::CollationElementIterator::NULLORDER
+        while ce != collation_element_iterator_class::NULLORDER
           p1 = (ce >> 24) & LAST_BYTE_MASK
           p2 = (ce >> 16) & LAST_BYTE_MASK
 

data/lib/twitter_cldr/resources/transform_test_importer.rb

@@ -3,17 +3,13 @@
 # Copyright 2012 Twitter, Inc
 # http://www.apache.org/licenses/LICENSE-2.0
 
-require 'java'
 require 'fileutils'
 
 module TwitterCldr
   module Resources
 
     # This class should be used with JRuby in 1.9 mode
-    class TransformTestImporter < IcuBasedImporter
-
-      attr_reader :output_file, :icu4j_path
-
+    class TransformTestImporter < Importer
       # most of these were taken from wikipedia, lol
       TEXT_SAMPLES = {
         latin: ["From today's featured article"],  # @TODO test capital letters,
@@ -23,7 +19,7 @@ module TwitterCldr
         gurmukhi: ["ਅੱਜ ਇਤਿਹਾਸ ਵਿੱਚ"],
         gujarati: ["આ માસનો ઉમદા લેખ"],
         bengali: ["নির্বাচিত নিবন্ধ"],
-        hangul: ["김창옥"],
+        hangul: ["김창옥", '모든 사용자는 위키백과에 직접 참여해 확인 가능'],
         arabic: ["مقالة اليوم المختارة"],
         han: ["因此只有两场风暴因造成"],
         hiragana: ["くろねこさま"],
@@ -37,7 +33,6 @@ module TwitterCldr
         malayalam: ['ഇടുക്കിയിലെ സൂര്യനെല്ലി സ്വദേശിനിയായ'],
         tamil: ['சென்னையில் வாழும் உலோ.செந்தமிழ்க்கோதை'],
         interindic: ['  '],
-        hangul: ['모든 사용자는 위키백과에 직접 참여해 확인 가능'],
         hebrew: ['על שמן של המיילדות במצרים, שפרה ופועה, נקראו'],
         simplified:  ['系统源于墨西哥以西的扰动天气区，并且位于更大规模的天气系统以内'],
         traditional: ['系統源於墨西哥以西的擾動天氣區，並且位於更大規模的天氣系統以內'],
@@ -50,15 +45,12 @@ module TwitterCldr
 
       BGN_SAMPLES = [:armenian, :katakana, :korean]
 
-      def initialize(output_file, icu4j_path)
-        @output_file = output_file
-        @icu4j_path = icu4j_path
-      end
+      requirement :icu, Versions.icu_version
+      output_path File.join(TwitterCldr::SPEC_DIR, 'transforms', 'test_data.yml')
+      ruby_engine :jruby
 
-      def import
-        require_icu4j(icu4j_path)
-
-        File.open(output_file, 'w+') do |f|
+      def execute
+        File.open(params.fetch(:output_path), 'w+') do |f|
           f.write(
             YAML.dump(
               generate_test_data(transformer.each_transform)
@@ -67,6 +59,8 @@ module TwitterCldr
         end
       end
 
+      private
+
       def generate_test_data(transforms)
         transforms.each_with_object([]) do |transform_id_str, ret|
           forward_id = transform_id.parse(transform_id_str)
@@ -104,8 +98,12 @@ module TwitterCldr
         TwitterCldr::Transforms::Transformer.exists?(id)
       end
 
+      def transliterator_class
+        @transliterator_class ||= requirements[:icu].get_class('com.ibm.icu.text.Transliterator')
+      end
+
       def generate_transform_samples(id, samples)
-        trans = com.ibm.icu.text.Transliterator.getInstance(id.to_s)
+        trans = transliterator_class.getInstance(id.to_s)
         samples.each_with_object({}) do |sample, ret|
           ret[sample] = trans.transliterate(sample)
         end
@@ -131,7 +129,7 @@ module TwitterCldr
       def transform_id
         TwitterCldr::Transforms::TransformId
       end
-
     end
+
   end
 end

data/lib/twitter_cldr/resources/uli/segment_exceptions_importer.rb

@@ -3,41 +3,42 @@
 # Copyright 2012 Twitter, Inc
 # http://www.apache.org/licenses/LICENSE-2.0
 
-require 'twitter_cldr/resources/download'
 require 'fileutils'
+require 'open-uri'
 require 'json'
 
 module TwitterCldr
   module Resources
     module Uli
-      class SegmentExceptionsImporter
+      class SegmentExceptionsImporter < Resources::Importer
 
         URL = "http://unicode.org/uli/trac/export/58/trunk/abbrs/json/%{locale}.json"
+        LOCALES = [:de, :en, :es, :fr, :it, :pt, :ru]   # these are the only locales ULI supports at the moment
 
-        attr_reader :input_path, :output_path
+        output_path 'uli/segments'
+        ruby_engine :mri
 
-        def initialize(input_path, output_path)
-          @input_path  = input_path
-          @output_path = output_path
-        end
-
-        def import(locales)
+        def execute
           FileUtils.mkdir_p(input_path)
           FileUtils.mkdir_p(output_path)
-          locales.each { |locale| import_locale(locale) }
+          LOCALES.each { |locale| import_locale(locale) }
         end
 
         private
 
+        def output_path
+          params.fetch(:output_path)
+        end
+
         def import_locale(locale)
           if input_file = download_resource_for(locale)
             output_file = File.join(output_path, "#{locale}.yml")
             exceptions = JSON.parse(File.read(input_file))
 
-            File.open(output_file, "w+") do |f|
+            File.open(output_file, 'w+') do |f|
               YAML.dump({
                 locale => {
-                  exceptions: exceptions["data"]["abbrs"]
+                  exceptions: exceptions['data']['abbrs']
                 }
               }, f)
             end
@@ -45,14 +46,25 @@ module TwitterCldr
         end
 
         def download_resource_for(locale)
-          input_file = input_path_for(locale)
-          TwitterCldr::Resources.download_if_necessary(
-            input_file, URL.gsub("%{locale}", locale.to_s)
-          )
+          input_file = input_file_for(locale)
+          url = URL % { locale: locale }
+
+          unless File.file?(input_file)
+            STDOUT.write("Downloading #{url}... ")
+            open(input_file, 'wb') { |file| file << open(url).read }
+            puts 'done'
+          end
+
           input_file
         end
 
-        def input_path_for(locale)
+        def input_path
+          @input_path ||= File.join(
+            TwitterCldr::VENDOR_DIR, 'uli', 'segments'
+          )
+        end
+
+        def input_file_for(locale)
           File.join(input_path, "#{locale}.json")
         end
 

data/lib/twitter_cldr/resources/unicode_data_importer.rb

@@ -3,52 +3,53 @@
 # Copyright 2012 Twitter, Inc
 # http://www.apache.org/licenses/LICENSE-2.0
 
-require 'twitter_cldr/resources/download'
-
 module TwitterCldr
   module Resources
 
-    class UnicodeDataImporter < UnicodeImporter
+    class UnicodeDataImporter < Importer
 
-      BLOCKS_URL           = 'ucd/Blocks.txt'
-      UNICODE_DATA_URL     = 'ucd/UnicodeData.txt'
-      CASEFOLDING_DATA_URL = 'ucd/CaseFolding.txt'
+      BLOCKS_FILE           = 'ucd/Blocks.txt'
+      UNICODE_DATA_FILE     = 'ucd/UnicodeData.txt'
+      CASEFOLDING_DATA_FILE = 'ucd/CaseFolding.txt'
 
-      # Arguments:
-      #
-      #   input_path  - path to a directory containing Blocks.txt and UnicodeData.txt
-      #   output_path - output directory for imported YAML files
-      #
-      def initialize(input_path, output_path)
-        @input_path  = input_path
-        @output_path = output_path
-      end
+      requirement :unicode, Versions.unicode_version, [BLOCKS_FILE, UNICODE_DATA_FILE, CASEFOLDING_DATA_FILE]
+      output_path 'unicode_data'
+      ruby_engine :mri
 
-      def import
+      def execute
         blocks           = import_blocks
         unicode_data     = import_unicode_data(blocks)
         casefolding_data = import_casefolding_data
 
-        File.open(File.join(@output_path, 'blocks.yml'), 'w') do |output|
+        STDOUT.write('Writing data to disk... ')
+
+        File.open(File.join(output_path, 'blocks.yml'), 'w') do |output|
           YAML.dump(blocks, output)
         end
 
-        FileUtils.mkdir_p(File.join(@output_path, 'blocks'))
+        FileUtils.mkdir_p(File.join(output_path, 'blocks'))
 
         unicode_data.each do |block_name, code_points|
-          File.open(File.join(@output_path, 'blocks', "#{block_name}.yml"), 'w') do |output|
+          File.open(File.join(output_path, 'blocks', "#{block_name}.yml"), 'w') do |output|
             YAML.dump(code_points, output)
           end
         end
 
-        File.open(File.join(@output_path, 'casefolding.yml'), 'w') do |output|
+        File.open(File.join(output_path, 'casefolding.yml'), 'w') do |output|
           YAML.dump(casefolding_data, output)
         end
+
+        puts 'done'
       end
 
       private
 
+      def output_path
+        params.fetch(:output_path)
+      end
+
       def import_blocks
+        STDOUT.write('Importing blocks... ')
         blocks = {}
 
         File.open(blocks_file) do |input|
@@ -62,48 +63,54 @@ module TwitterCldr
           end
         end
 
+        puts 'done'
         blocks
       end
 
+      def parse_file(file, &block)
+        UnicodeFileParser.parse_standard_file(file, &block)
+      end
+
       def import_unicode_data(blocks)
+        STDOUT.write('Importing Unicode data... ')
         unicode_data = Hash.new do |hash, key|
           hash[key] = Hash.new { |h, k| h[k] = {} }
         end
 
-        parse_standard_file(unicode_data_file) do |data|
+        parse_file(unicode_data_file) do |data|
           data[0] = data[0].hex
           unicode_data[find_block(blocks, data[0]).first][data[0]] = data
         end
 
+        puts 'done'
         unicode_data
       end
 
       def import_casefolding_data
-        parse_standard_file(casefold_data_file).map do |data|
+        STDOUT.write('Importing casefolding data... ')
+
+        casefolding_data = parse_file(casefold_data_file).map do |data|
           {
             source: data[0].hex,
             target: data[2].split(" ").map(&:hex),
             status: data[1]
           }
         end
+
+        puts 'done'
+        casefolding_data
       end
 
       def casefold_data_file
-        TwitterCldr::Resources.download_unicode_data_if_necessary(
-          File.join(@input_path, 'CaseFolding.txt'), CASEFOLDING_DATA_URL
-        )
+        requirements[:unicode].source_path_for(CASEFOLDING_DATA_FILE)
       end
 
       def unicode_data_file
-        TwitterCldr::Resources.download_unicode_data_if_necessary(
-          File.join(@input_path, 'UnicodeData.txt'), UNICODE_DATA_URL
-        )
+        requirements[:unicode].source_path_for(UNICODE_DATA_FILE)
       end
 
       def blocks_file
-        TwitterCldr::Resources.download_unicode_data_if_necessary(
-          File.join(@input_path, 'Blocks.txt'), BLOCKS_URL
-        )
+        requirements[:unicode].source_path_for(BLOCKS_FILE)
       end
 
       def find_block(blocks, code_point)