twitter_cldr 4.0.0 → 4.1.0

data/lib/twitter_cldr/resources/casefolder_class_generator.rb

@@ -7,24 +7,35 @@ require 'erb'
 
 module TwitterCldr
   module Resources
-    class CasefolderClassGenerator
+    class CasefolderClassGenerator < Importer
 
-      attr_reader :template_file, :output_dir
+      requirement :dependency, [UnicodeDataImporter]
+      output_path File.join(TwitterCldr::LIB_DIR, 'twitter_cldr', 'shared')
+      parameter :template_file, File.join(TwitterCldr::LIB_DIR, 'twitter_cldr', 'resources', 'casefolder.rb.erb')
+      ruby_engine :mri
 
-      def initialize(template_file, output_dir)
-        @template_file = template_file
-        @output_dir = output_dir
-      end
+      private
 
-      def generate
-        output_file = File.basename(template_file).chomp(".erb")
-        File.open(File.join(output_dir, output_file), "w+") do |f|
+      def execute
+        File.open(output_path, 'w+') do |f|
           f.write(
             ERB.new(File.read(template_file)).result(binding)
           )
         end
       end
 
+      def template_file
+        params.fetch(:template_file)
+      end
+
+      def output_path
+        File.join(params.fetch(:output_path), output_file)
+      end
+
+      def output_file
+        File.basename(template_file).chomp('.erb')
+      end
+
       def casefolding_char_class_for(status)
         to_regex_char_sequence(casefolding_data_for(status))
       end
@@ -48,8 +59,6 @@ module TwitterCldr
         str << "\n#{"  " * (indent - 1)}}"
       end
 
-      private
-
       def to_regex_char_sequence(casefold_data)
         casefold_data.map { |(source, _)| to_utf8(source) }.join("|")
       end
@@ -67,9 +76,9 @@ module TwitterCldr
       end
 
       def resource
-        @@resource ||= TwitterCldr.get_resource("unicode_data", "casefolding")
+        @resource ||= TwitterCldr.get_resource('unicode_data', 'casefolding')
       end
 
     end
   end
-end
+end

data/lib/twitter_cldr/resources/collation_tries_importer.rb

@@ -0,0 +1,44 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Resources
+    class CollationTriesImporter < Importer
+      requirement :dependency, [TailoringImporter]
+      locales TwitterCldr.supported_locales
+      ruby_engine :jruby
+
+      private
+
+      def execute
+        update_default_trie_dump
+
+        params.fetch(:locales).each do |locale|
+          update_tailoring_trie_dump(locale)
+        end
+      end
+
+      private
+
+      def update_default_trie_dump
+        save_trie_dump(TwitterCldr::Collation::TrieLoader::DEFAULT_TRIE_LOCALE, default_trie)
+      end
+
+      def update_tailoring_trie_dump(locale)
+        save_trie_dump(locale, TwitterCldr::Collation::TrieBuilder.load_tailored_trie(locale, @default_trie))
+      end
+
+      def save_trie_dump(locale, trie)
+        path = TwitterCldr::Collation::TrieLoader.dump_path(locale)
+        FileUtils.mkdir_p(File.dirname(path))
+        File.write(path, Marshal.dump(trie))
+      end
+
+      def default_trie
+        @default_trie ||= TwitterCldr::Collation::TrieBuilder.load_default_trie
+      end
+    end
+  end
+end

data/lib/twitter_cldr/resources/hyphenation_importer.rb

@@ -6,28 +6,19 @@
 module TwitterCldr
   module Resources
 
-    class HyphenationImporter
+    class HyphenationImporter < Importer
+      GIT_SHA = '0d3b5e5314e68c3cf5d573b2e7bdc11143dcb821'
       REPO_URL = 'git@github.com:LibreOffice/dictionaries.git'
       ENCODING_MAP = {
         'microsoft-cp1251' => Encoding::Windows_1251
       }
 
-      # Arguments:
-      #
-      #   input_path  - path to a directory to clone hyphenation data into
-      #   output_path - output directory for generated YAML files
-      #   ref         - git ref to use
-      #
-      def initialize(input_path, output_path, ref)
-        @input_path  = input_path
-        @output_path = output_path
-        @ref = ref
-      end
+      requirement :git, REPO_URL, GIT_SHA
+      output_path 'shared/hyphenation'
+      ruby_engine :mri
 
-      def import
-        FileUtils.mkdir_p(@input_path)
-        FileUtils.mkdir_p(@output_path)
-        clone_or_fetch_if_necessary
+      def execute
+        FileUtils.mkdir_p(output_path)
 
         each_dictionary do |path, locale|
           import_dictionary(path, locale)
@@ -69,7 +60,7 @@ module TwitterCldr
         options.delete(:encoding)
 
         File.write(
-          File.join(@output_path, "#{locale}.yml"),
+          File.join(output_path, "#{locale}.yml"),
           YAML.dump({ options: options, rules: rules })
         )
       end
@@ -78,37 +69,20 @@ module TwitterCldr
         ENCODING_MAP.fetch(encoding.downcase, encoding)
       end
 
-      def each_dictionary
-        Dir.glob(File.join(@input_path, '**/hyph_*.dic')) do |path|
-          locale = TwitterCldr::Shared::Locale.parse(File.basename(path)[5..-5])
-          yield path, locale.dasherized
-        end
-      end
-
-      def dictionary_path
-        File.join(@input_path, 'dictionaries')
+      def source_path
+        requirements[:git].source_path
       end
 
-      def clone_or_fetch_if_necessary
-        if File.exist?(@input_path)
-          unless ref_exists?
-            in_repo { `git fetch` }
-          end
-        else
-          `git clone #{REPO_URL} #{@input_path}`
-        end
+      def output_path
+        params.fetch(:output_path)
       end
 
-      def ref_exists?
-        in_repo do
-          `git rev-parse --verify --quiet #{@ref}`
-          $?.exitstatus == 0
+      def each_dictionary
+        Dir.glob(File.join(source_path, '**/hyph_*.dic')) do |path|
+          locale = TwitterCldr::Shared::Locale.parse(File.basename(path)[5..-5])
+          yield path, locale.dasherized
         end
       end
-
-      def in_repo(&block)
-        Dir.chdir(@input_path, &block)
-      end
     end
 
   end

data/lib/twitter_cldr/resources/import_resolver.rb

@@ -0,0 +1,71 @@
+require 'tsort'
+
+module TwitterCldr
+  module Resources
+    class ImportResolver
+      include TSort
+
+      attr_reader :importers
+
+      def initialize(importers = Resources.importer_classes_for_ruby_engine)
+        @importers = importers
+      end
+
+      def import
+        check_unmet_deps
+        import_in_order
+      end
+
+      private
+
+      def import_in_order
+        tsort.each { |instance| instance.import }
+      end
+
+      def tsort_each_node(&block)
+        instances.each(&block)
+      end
+
+      def tsort_each_child(instance, &block)
+        deps_for(instance).map do |dep_class|
+          yield instances.find { |ins| ins.class == dep_class }
+        end
+      end
+
+      def check_unmet_deps
+        instances.each do |instance|
+          check_unmet_instance_deps(instance)
+        end
+      end
+
+      def check_unmet_instance_deps(instance)
+        unmet_deps = unmet_deps_for(instance)
+
+        unless unmet_deps.empty?
+          list = unmet_deps.map { |d| d.name }.join(', ')
+          raise "#{instance.class} dependencies are not met: #{list}"
+        end
+      end
+
+      def instances
+        @instances ||= importers.map do |importer|
+          importer.is_a?(Class) ? importer.new : importer
+        end
+      end
+
+      def unmet_deps_for(instance)
+        deps_for(instance).reject do |dep_class|
+          instances.any? { |ins| ins.class == dep_class }
+        end
+      end
+
+      def deps_for(instance)
+        if dep_requirement = instance.requirements[:dependency]
+          dep_requirement.importer_classes
+        else
+          []
+        end
+      end
+    end
+  end
+end

data/lib/twitter_cldr/resources/importer.rb

@@ -0,0 +1,107 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Resources
+
+    class Importer
+      DEFAULT_ENGINE = :mri
+
+      class << self
+        def requirement(name, *args)
+          const_name = "#{name.to_s.capitalize}Requirement".to_sym
+          requirements[name] = Requirements.const_get(const_name).new(*args)
+        end
+
+        def ruby_engine(engine)
+          @ruby_engine = engine
+        end
+
+        def output_path(path)
+          @output_path = if path.start_with?('/')
+            path
+          else
+            File.join(TwitterCldr::RESOURCES_DIR, path)
+          end
+        end
+
+        def locales(locs)
+          @locales = locs
+        end
+
+        def parameter(key, value)
+          parameters[key] = value
+        end
+
+        def default_params
+          parameters.merge(
+            output_path: @output_path,
+            locales: @locales,
+            ruby_engine: @ruby_engine || DEFAULT_ENGINE
+          )
+        end
+
+        def requirements
+          @requirements ||= {}
+        end
+
+        def parameters
+          @parameters ||= {}
+        end
+      end
+
+      attr_reader :params, :requirements
+
+      def initialize(options = {})
+        @params = self.class.default_params.merge(options)
+        @requirements = self.class.requirements
+      end
+
+      def can_import?
+        importability_errors.empty?
+      end
+
+      def import
+        if can_import?
+          prepare
+          execute
+        else
+          raise "Can't import #{self.class.name}: #{importability_errors.first}"
+        end
+      end
+
+      def prepare
+        before_prepare
+        requirements.each { |_, req| req.prepare }
+        after_prepare
+      end
+
+      private
+
+      def importability_errors
+        @importability_errors ||= [].tap do |errors|
+          errors << 'incompatible RUBY_ENGINE' unless compatible_engine?
+        end
+      end
+
+      def compatible_engine?
+        case params.fetch(:ruby_engine)
+          when :mri
+            RUBY_ENGINE == 'ruby'
+          when :jruby
+            RUBY_ENGINE == 'jruby'
+          else
+            false
+        end
+      end
+
+      def before_prepare
+      end
+
+      def after_prepare
+      end
+    end
+  end
+end

data/lib/twitter_cldr/resources/language_codes_importer.rb

@@ -3,12 +3,12 @@
 # Copyright 2012 Twitter, Inc
 # http://www.apache.org/licenses/LICENSE-2.0
 
-require 'twitter_cldr/resources/download'
+require 'open-uri'
 
 module TwitterCldr
   module Resources
 
-    class LanguageCodesImporter
+    class LanguageCodesImporter < Importer
 
       BCP_47_FILE, ISO_639_FILE = %w[bcp-47.txt iso-639.txt]
 
@@ -26,42 +26,39 @@ module TwitterCldr
         bcp_47_alt:     :bcp_47
       }
 
-      def initialize(input_path, output_path)
-        @input_path  = input_path
-        @output_path = output_path
-      end
+      output_path 'shared'
+      ruby_engine :mri
+
+      private
 
-      def import(import_yaml = false)
+      def execute
         prepare_data
-        import_data(import_yaml)
+        import_data
       end
 
-      private
-
       def prepare_data
         INPUT_DATA.each do |file, url|
-          TwitterCldr::Resources.download_if_necessary(File.join(@input_path, file), url)
+          source_path = source_path_for(file)
+          open(source_path, 'wb') { |file| file << open(url).read }
         end
       end
 
-      def import_data(import_yaml)
+      def source_path_for(file)
+        File.join(TwitterCldr::VENDOR_DIR, file)
+      end
+
+      def import_data
         result = import_iso_639
         result = import_bcp_47(result)
 
         language_codes = Hash[result.inject({}) { |memo, (key, value)| memo[key] = Hash[value.sort]; memo }.sort]
-
         language_codes_table = build_table(language_codes)
 
-        write('language_codes_table.dump', 'wb', Marshal.dump(language_codes_table))
-
-        if import_yaml
-          write('language_codes.yml', 'w:utf-8', YAML.dump(language_codes))
-          write('language_codes_table.yml', 'w:utf-8', YAML.dump(language_codes_table))
-        end
+        write('language_codes_table.dump', Marshal.dump(language_codes_table))
       end
 
-      def write(file, mode, data)
-        File.open(File.join(@output_path, file), mode) { |output| output.write(data) }
+      def write(file, data)
+        File.write(File.join(params.fetch(:output_path), file), data)
       end
 
       # Generates codes in the following format:
@@ -76,8 +73,8 @@ module TwitterCldr
       # }
       #
       def import_iso_639(result = {})
-        File.open(File.join(@input_path, ISO_639_FILE)) do |file|
-          lines = file.lines
+        File.open(source_path_for(ISO_639_FILE)) do |file|
+          lines = file.each_line
           lines.next # skip header
 
           lines.each do |line|
@@ -120,8 +117,8 @@ module TwitterCldr
       #   }
       # }
       def import_bcp_47(result = {})
-        File.open(File.join(@input_path, BCP_47_FILE)) do |file|
-          lines = file.lines
+        File.open(source_path_for(BCP_47_FILE)) do |file|
+          lines = file.each_line
           lines.next # skip header
 
           data  = {}
@@ -212,21 +209,21 @@ module TwitterCldr
       end
 
       ISO_639_COLUMNS = [
-          :code,            # Code
-          :status,          # Status
-          :partner_agency,  # Partner Agency
-          :iso_639_3,       # 639_3
-          :iso_639_2,       # 639_2 (alpha-3 bibliographic/terminology code)
-          :b_code,          # alpha-3 bibliographic code if iso_639_2 contains terminology code
-          :bt_equiv,        # bt_equiv (alpha-3 bibliographic/terminology equivalent)
-          :iso_639_1,       # 639_1
-          :name,            # Reference_Name
-          :scope,           # Element_Scope
-          :type,            # Language_Type
-          :docs             # Documentation
+        :code,            # Code
+        :status,          # Status
+        :partner_agency,  # Partner Agency
+        :iso_639_3,       # 639_3
+        :iso_639_2,       # 639_2 (alpha-3 bibliographic/terminology code)
+        :b_code,          # alpha-3 bibliographic code if iso_639_2 contains terminology code
+        :bt_equiv,        # bt_equiv (alpha-3 bibliographic/terminology equivalent)
+        :iso_639_1,       # 639_1
+        :name,            # Reference_Name
+        :scope,           # Element_Scope
+        :type,            # Language_Type
+        :docs             # Documentation
       ]
 
     end
 
   end
-end
+end