twitter_cldr 3.2.1 → 3.3.0

data/lib/twitter_cldr/resources/properties/property_importer.rb

@@ -0,0 +1,83 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'twitter_cldr/resources/download'
+
+module TwitterCldr
+  module Resources
+    module Properties
+
+      class PropertyImporter < UnicodeImporter
+        attr_reader :input_path, :property_name
+        attr_reader :data_url, :data_path, :database
+
+        def initialize(options)
+          @input_path = options.fetch(:input_path)
+          @property_name = options.fetch(:property_name)
+          @data_url = options.fetch(:data_url)
+          @data_path = options.fetch(:data_path)
+          @database = options.fetch(:database)
+        end
+
+        def import
+          load.each_pair do |property_name, property_values|
+            property_values.each_pair do |property_value, ranges|
+              database.store(property_name, property_value, ranges)
+            end
+          end
+        end
+
+        private
+
+        def load
+          results = Hash.new do |h, k|
+            h[k] = Hash.new { |h, k| h[k] = [] }
+          end
+
+          rangify_hash(
+            parse_standard_file(data_file).each_with_object(results) do |data, ret|
+              next unless data[0].size > 0
+
+              if block_given?
+                yield data, ret
+              else
+                code_points = expand_range(data[0])
+                property_value = format_property_value(data[1])
+                ret[property_name][property_value] += code_points
+              end
+            end
+          )
+        end
+
+        def rangify_hash(hash)
+          hash.each_with_object({}) do |(key, value), ret|
+            ret[key] = case value
+              when Hash
+                rangify_hash(value)
+              when Array
+                TwitterCldr::Utils::RangeSet.from_array(value)
+            end
+          end
+        end
+
+        def expand_range(str)
+          initial, final = str.split("..")
+          (initial.to_i(16)..(final || initial).to_i(16)).to_a
+        end
+
+        def format_property_value(value)
+          value
+        end
+
+        def data_file
+          TwitterCldr::Resources.download_unicode_data_if_necessary(
+            File.join(input_path, data_path), data_url
+          )
+        end
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/resources/properties/script_extensions_property_importer.rb

@@ -0,0 +1,40 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Resources
+    module Properties
+
+      class ScriptExtensionsPropertyImporter < PropertyImporter
+        DATA_URL = 'ucd/ScriptExtensions.txt'
+        PROPERTY_NAME = 'Script_Extensions'
+
+        def initialize(input_path, database)
+          super(
+            input_path: input_path,
+            property_name: PROPERTY_NAME,
+            data_url: DATA_URL,
+            data_path: File.basename(DATA_URL),
+            database: database
+          )
+        end
+
+        private
+
+        def load
+          super do |data, ret|
+            code_points = expand_range(data[0])
+            property_values = data[1].split(' ')
+
+            property_values.each do |property_value|
+              ret[PROPERTY_NAME][property_value] += code_points
+            end
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/resources/properties/script_property_importer.rb

@@ -0,0 +1,27 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Resources
+    module Properties
+
+      class ScriptPropertyImporter < PropertyImporter
+        DATA_URL = 'ucd/Scripts.txt'
+        PROPERTY_NAME = 'Script'
+
+        def initialize(input_path, database)
+          super(
+            input_path: input_path,
+            property_name: PROPERTY_NAME,
+            data_url: DATA_URL,
+            data_path: File.basename(DATA_URL),
+            database: database
+          )
+        end
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/resources/properties/sentence_break_property_importer.rb

@@ -0,0 +1,27 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Resources
+    module Properties
+
+      class SentenceBreakPropertyImporter < PropertyImporter
+        DATA_URL = 'ucd/auxiliary/SentenceBreakProperty.txt'
+        PROPERTY_NAME = 'Sentence_Break'
+
+        def initialize(input_path, database)
+          super(
+            input_path: input_path,
+            property_name: PROPERTY_NAME,
+            data_url: DATA_URL,
+            data_path: File.basename(DATA_URL),
+            database: database
+          )
+        end
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/resources/properties/unicode_data_properties_importer.rb

@@ -0,0 +1,60 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Resources
+    module Properties
+
+      class UnicodeDataPropertiesImporter < PropertyImporter
+        DATA_URL = 'ucd/UnicodeData.txt'
+
+        PROPERTIES = {
+          2  => 'General_Category',
+          3  => 'Canonical_Combining_Class',
+          4  => 'Bidi_Class',
+          7  => 'Numeric_Type',
+          9  => 'Bidi_Mirrored',
+        }
+
+        def initialize(input_path, database)
+          super(
+            input_path: input_path,
+            property_name: nil,
+            data_url: DATA_URL,
+            data_path: File.basename(DATA_URL),
+            database: database
+          )
+        end
+
+        private
+
+        def load
+          range_start = nil
+
+          super do |data, ret|
+            code_points = expand_range(data[0])
+
+            # UnicodeData.txt can contain ranges of characters
+            # specified with "First" and "Last" identifiers in
+            # the name field.
+            if data[1].include?(', First')
+              range_start = code_points.first
+              next
+            elsif data[1].include?(', Last')
+              code_points = (range_start..code_points.first).to_a
+              range_start = nil
+            end
+
+            PROPERTIES.each_pair do |idx, property_name|
+              property_value = format_property_value(data[idx])
+              ret[property_name][property_value] += code_points
+            end
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/resources/properties/word_break_property_importer.rb

@@ -0,0 +1,27 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Resources
+    module Properties
+
+      class WordBreakPropertyImporter < PropertyImporter
+        DATA_URL = 'ucd/auxiliary/WordBreakProperty.txt'
+        PROPERTY_NAME = 'Word_Break'
+
+        def initialize(input_path, database)
+          super(
+            input_path: input_path,
+            property_name: PROPERTY_NAME,
+            data_url: DATA_URL,
+            data_path: File.basename(DATA_URL),
+            database: database
+          )
+        end
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/resources/properties.rb

@@ -0,0 +1,36 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'twitter_cldr/resources/download'
+
+module TwitterCldr
+  module Resources
+    module Properties
+
+      autoload :PropertiesImporter,                      'twitter_cldr/resources/properties/properties_importer'
+      autoload :PropertyImporter,                        'twitter_cldr/resources/properties/property_importer'
+      autoload :AgePropertyImporter,                     'twitter_cldr/resources/properties/age_property_importer'
+      autoload :ArabicShapingPropertyImporter,           'twitter_cldr/resources/properties/arabic_shaping_property_importer'
+      autoload :BidiBracketsPropertyImporter,            'twitter_cldr/resources/properties/bidi_brackets_property_importer'
+      autoload :BidiMirroringPropertyImporter,           'twitter_cldr/resources/properties/bidi_mirroring_property_importer'
+      autoload :BlocksPropertyImporter,                  'twitter_cldr/resources/properties/blocks_property_importer'
+      autoload :DerivedCorePropertiesImporter,           'twitter_cldr/resources/properties/derived_core_properties_importer'
+      autoload :EastAsianWidthPropertyImporter,          'twitter_cldr/resources/properties/east_asian_width_property_importer'
+      autoload :GraphemeBreakPropertyImporter,           'twitter_cldr/resources/properties/grapheme_break_property_importer'
+      autoload :HangulSyllableTypePropertyImporter,      'twitter_cldr/resources/properties/hangul_syllable_type_property_importer'
+      autoload :IndicPositionalCategoryPropertyImporter, 'twitter_cldr/resources/properties/indic_positional_category_property_importer'
+      autoload :IndicSyllabicCategoryPropertyImporter,   'twitter_cldr/resources/properties/indic_syllabic_category_property_importer'
+      autoload :JamoPropertyImporter,                    'twitter_cldr/resources/properties/jamo_property_importer'
+      autoload :LineBreakPropertyImporter,               'twitter_cldr/resources/properties/line_break_property_importer'
+      autoload :PropListImporter,                        'twitter_cldr/resources/properties/prop_list_importer'
+      autoload :ScriptExtensionsPropertyImporter,        'twitter_cldr/resources/properties/script_extensions_property_importer'
+      autoload :ScriptPropertyImporter,                  'twitter_cldr/resources/properties/script_property_importer'
+      autoload :SentenceBreakPropertyImporter,           'twitter_cldr/resources/properties/sentence_break_property_importer'
+      autoload :UnicodeDataPropertiesImporter,           'twitter_cldr/resources/properties/unicode_data_properties_importer'
+      autoload :WordBreakPropertyImporter,               'twitter_cldr/resources/properties/word_break_property_importer'
+
+    end
+  end
+end

data/lib/twitter_cldr/resources/readme_renderer.rb

@@ -35,8 +35,8 @@ module TwitterCldr
 
       def assert(got, expected)
         if got.is_a?(String) && expected.is_a?(String)
-          got = got.localize.normalize(:using => :NFKC).to_s
-          expected = expected.localize.normalize(:using => :NFKC).to_s
+          got = got.localize.normalize(using: :NFKC).to_s
+          expected = expected.localize.normalize(using: :NFKC).to_s
         end
 
         unless objs_equal?(got, expected)

data/lib/twitter_cldr/resources/segment_tests_importer.rb

@@ -0,0 +1,66 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'twitter_cldr/resources/download'
+require 'fileutils'
+
+module TwitterCldr
+  module Resources
+    class SegmentTestsImporter < UnicodeImporter
+
+      URL_ROOT = "ucd/auxiliary"
+      TEST_FILES = [
+        'WordBreakTest.txt', 'SentenceBreakTest.txt'
+      ]
+
+      attr_reader :input_path, :output_path
+
+      def initialize(input_path, output_path)
+        @input_path  = input_path
+        @output_path = output_path
+      end
+
+      def import
+        FileUtils.mkdir_p(input_path)
+        FileUtils.mkdir_p(output_path)
+
+        TEST_FILES.each do |test_file|
+          import_test_file(test_file)
+        end
+      end
+
+      private
+
+      def import_test_file(test_file)
+        url = "#{URL_ROOT}/#{test_file}"
+        input_file = input_file_for(test_file)
+        output_file = output_path_for(test_file)
+        download(input_file, url)
+        result = parse_standard_file(input_file).map(&:first)
+        File.write(output_file, YAML.dump(result))
+      end
+
+      def input_file_for(test_file)
+        File.join(input_path, test_file)
+      end
+
+      def output_path_for(test_file)
+        base = underscore(test_file.chomp(File.extname(test_file)))
+        File.join(output_path, "#{base}.yml")
+      end
+
+      def download(input_file, url)
+        TwitterCldr::Resources.download_unicode_data_if_necessary(
+          input_file, url
+        )
+      end
+
+      def underscore(str)
+        str.gsub(/(.)([A-Z])/, '\1_\2').downcase
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/resources/tailoring_importer.rb

@@ -33,9 +33,9 @@ module TwitterCldr
       }
 
       EMPTY_TAILORING_DATA = {
-        :collator_options => {},
-        :tailored_table => '',
-        :suppressed_contractions => ''
+        collator_options: {},
+        tailored_table: '',
+        suppressed_contractions: ''
       }
 
       class ImportError < RuntimeError; end
@@ -117,9 +117,9 @@ module TwitterCldr
         end
 
         {
-            :collator_options        => parse_collator_options(collation_rules),
-            :tailored_table          => parse_tailorings(collation_rules, locale),
-            :suppressed_contractions => parse_suppressed_contractions(collation_rules)
+            collator_options:        parse_collator_options(collation_rules),
+            tailored_table:          parse_tailorings(collation_rules, locale),
+            suppressed_contractions: parse_suppressed_contractions(collation_rules)
         }
       end
 
@@ -232,4 +232,4 @@ module TwitterCldr
     end
 
   end
-end
+end

data/lib/twitter_cldr/resources/uli/segment_exceptions_importer.rb

@@ -37,7 +37,7 @@ module TwitterCldr
             File.open(output_file, "w+") do |f|
               YAML.dump({
                 locale => {
-                  :exceptions => exceptions["data"]["abbrs"]
+                  exceptions: exceptions["data"]["abbrs"]
                 }
               }, f)
             end

data/lib/twitter_cldr/resources/unicode_data_importer.rb

@@ -10,9 +10,9 @@ module TwitterCldr
 
     class UnicodeDataImporter < UnicodeImporter
 
-      BLOCKS_URL           = 'ftp://ftp.unicode.org/Public/UNIDATA/Blocks.txt'
-      UNICODE_DATA_URL     = 'ftp://ftp.unicode.org/Public/UNIDATA/UnicodeData.txt'
-      CASEFOLDING_DATA_URL = 'ftp://ftp.unicode.org/Public/UNIDATA/CaseFolding.txt'
+      BLOCKS_URL           = 'ucd/Blocks.txt'
+      UNICODE_DATA_URL     = 'ucd/UnicodeData.txt'
+      CASEFOLDING_DATA_URL = 'ucd/CaseFolding.txt'
 
       # Arguments:
       #
@@ -28,9 +28,6 @@ module TwitterCldr
         blocks           = import_blocks
         unicode_data     = import_unicode_data(blocks)
         casefolding_data = import_casefolding_data
-        index_data       = build_indices(
-          TwitterCldr::Shared::CodePoint::INDICES, unicode_data
-        )
 
         File.open(File.join(@output_path, 'blocks.yml'), 'w') do |output|
           YAML.dump(blocks, output)
@@ -47,28 +44,6 @@ module TwitterCldr
         File.open(File.join(@output_path, 'casefolding.yml'), 'w') do |output|
           YAML.dump(casefolding_data, output)
         end
-
-        FileUtils.mkdir_p(File.join(@output_path, 'indices'))
-
-        indices = index_data
-
-        indices.each_pair do |index_name, data|
-          File.open(File.join(@output_path, "indices", "#{index_name}.yml"), 'w') do |output|
-            YAML.dump(data, output)
-          end
-        end
-
-        File.open(File.join(@output_path, "indices", "keys.yml"), 'w') do |output|
-          YAML.dump(
-            indices.inject({}) do |ret, (index_name, data)|
-              data.keys.each do |prop|
-                ret[prop] ||= []
-                ret[prop] << index_name
-              end
-              ret
-            end, output
-          )
-        end
       end
 
       private
@@ -91,7 +66,9 @@ module TwitterCldr
       end
 
       def import_unicode_data(blocks)
-        unicode_data = Hash.new { |hash, key| hash[key] = Hash.new { |h, k| h[k] = {} } }
+        unicode_data = Hash.new do |hash, key|
+          hash[key] = Hash.new { |h, k| h[k] = {} }
+        end
 
         parse_standard_file(unicode_data_file) do |data|
           data[0] = data[0].hex
@@ -104,47 +81,29 @@ module TwitterCldr
       def import_casefolding_data
         parse_standard_file(casefold_data_file).map do |data|
           {
-            :source => data[0].hex,
-            :target => data[2].split(" ").map(&:hex),
-            :status => data[1]
+            source: data[0].hex,
+            target: data[2].split(" ").map(&:hex),
+            status: data[1]
           }
         end
       end
 
-      def build_indices(indices, unicode_data)
-        indices.inject({}) do |index_ret, index_name|
-          field_index = TwitterCldr::Shared::CODE_POINT_FIELDS.find_index do |field|
-            field == index_name
-          end
-
-          index_ret[index_name] = Hash.new { |hash, key| hash[key] = [] }
-
-          unicode_data.each_pair do |block_name, block_data|
-            block_data.each_pair do |code_point, data|
-              index_ret[index_name][data[field_index].to_sym] << code_point
-            end
-          end
-
-          index_ret
-        end.inject({}) do |index_ret, (index_key, index_data)|
-          index_ret[index_key] = index_data.inject({}) do |field_ret, (field_key, field_data)|
-            field_ret[field_key] = TwitterCldr::Utils::RangeSet.rangify(field_data)
-            field_ret
-          end
-          index_ret
-        end
-      end
-
       def casefold_data_file
-        TwitterCldr::Resources.download_if_necessary(File.join(@input_path, 'CaseFolding.txt'), CASEFOLDING_DATA_URL)
+        TwitterCldr::Resources.download_unicode_data_if_necessary(
+          File.join(@input_path, 'CaseFolding.txt'), CASEFOLDING_DATA_URL
+        )
       end
 
       def unicode_data_file
-        TwitterCldr::Resources.download_if_necessary(File.join(@input_path, 'UnicodeData.txt'), UNICODE_DATA_URL)
+        TwitterCldr::Resources.download_unicode_data_if_necessary(
+          File.join(@input_path, 'UnicodeData.txt'), UNICODE_DATA_URL
+        )
       end
 
       def blocks_file
-        TwitterCldr::Resources.download_if_necessary(File.join(@input_path, 'Blocks.txt'), BLOCKS_URL)
+        TwitterCldr::Resources.download_unicode_data_if_necessary(
+          File.join(@input_path, 'Blocks.txt'), BLOCKS_URL
+        )
       end
 
       def find_block(blocks, code_point)
@@ -158,4 +117,4 @@ module TwitterCldr
     end
 
   end
-end
+end

data/lib/twitter_cldr/resources/unicode_property_aliases_importer.rb

@@ -0,0 +1,97 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'twitter_cldr/resources/download'
+
+module TwitterCldr
+  module Resources
+
+    class UnicodePropertyAliasesImporter < UnicodeImporter
+      PROPERTY_ALIASES_URL = 'ucd/PropertyAliases.txt'
+      PROPERTY_VALUE_ALIASES_URL = 'ucd/PropertyValueAliases.txt'
+
+      # Arguments:
+      #
+      #   input_path  - path to a directory containing Scripts.txt
+      #   output_path - output directory for imported YAML files
+      #
+      def initialize(input_path, output_path)
+        @input_path  = input_path
+        @output_path = output_path
+      end
+
+      def import
+        File.write(
+          File.join(@output_path, 'property_value_aliases.yml'),
+          YAML.dump(parse_property_value_aliases)
+        )
+
+        File.write(
+          File.join(@output_path, 'property_aliases.yml'),
+          YAML.dump(parse_property_aliases)
+        )
+      end
+
+      private
+
+      def parse_property_aliases
+        Hash.new { |h, k| h[k] = [] }.tap do |result|
+          parse_standard_file(property_aliases_data_file) do |data|
+            property = data[0]
+            result[property] = parse_alias(data)
+          end
+        end
+      end
+
+      def parse_property_value_aliases
+        Hash.new { |h, k| h[k] = [] }.tap do |result|
+          parse_standard_file(property_value_aliases_data_file) do |data|
+            property_value = data[0]
+            result[property_value] << if property_value == 'ccc'
+              parse_ccc_value_alias(data)
+            else
+              parse_value_alias(data)
+            end
+          end
+        end
+      end
+
+      def parse_alias(data)
+        {
+          long_name: data[1],
+          additional: data[2..-1]
+        }
+      end
+
+      def parse_value_alias(data)
+        {
+          abbreviated_name: data[1],
+          long_name: data[2]
+        }
+      end
+
+      def parse_ccc_value_alias(data)
+        {
+          numeric: data[1],  # don't know what this means
+          abbreviated_name: data[2],
+          long_name: data[3]
+        }
+      end
+
+      def property_aliases_data_file
+        TwitterCldr::Resources.download_unicode_data_if_necessary(
+          File.join(@input_path, 'PropertyAliases.txt'), PROPERTY_ALIASES_URL
+        )
+      end
+
+      def property_value_aliases_data_file
+        TwitterCldr::Resources.download_unicode_data_if_necessary(
+          File.join(@input_path, 'PropertyValueAliases.txt'), PROPERTY_VALUE_ALIASES_URL
+        )
+      end
+    end
+
+  end
+end