twitter_cldr 6.9.0 → 6.11.2

data/lib/twitter_cldr/resources/cldr_dtd.rb

@@ -0,0 +1,110 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Resources
+
+    class CldrDTD
+      class Attr
+        attr_reader :name, :element_name, :dtd
+
+        def initialize(name, element_name, dtd)
+          @name = name
+          @element_name = element_name
+          @dtd = dtd
+        end
+
+        def values
+          @values ||= begin
+            attr_line_idx = schema.find_index do |line|
+              line.include?("<!ATTLIST #{element_name} #{name} ")
+            end
+
+            return [] unless attr_line_idx
+
+            attr_line = schema[attr_line_idx]
+
+            if comment = find_match_comment_after(attr_line_idx + 1)
+              parse_match(comment)
+            else
+              start_idx = attr_line.index('(')
+              return [] unless start_idx
+
+              finish_idx = attr_line.rindex(')')
+              attr_line[(start_idx + 1)...finish_idx].split('|').map(&:strip)
+            end
+          end
+        end
+
+        private
+
+        def find_match_comment_after(idx)
+          loop do
+            return nil if idx > schema.size
+
+            if schema[idx].strip.start_with?('<!--@MATCH')
+              break
+            elsif schema[idx].strip.start_with?('<!--')
+              idx += 1
+            else
+              return nil
+            end
+          end
+
+          schema[idx]
+        end
+
+        def parse_match(str)
+          m = str.match(/<!--@MATCH:([^\/]+)\/(.*)-->/)
+          return [] unless m
+
+          type, args = m.captures
+
+          case type
+            when 'literal'
+              args.split(',').map(&:strip)
+            when 'range'
+              start, finish = args.split('~')
+              ((start.to_i)..(finish.to_i)).to_a
+          end
+        end
+
+        def schema
+          dtd.schema
+        end
+      end
+
+      attr_reader :cldr_requirement
+
+      def initialize(cldr_requirement)
+        @cldr_requirement = cldr_requirement
+      end
+
+      def find_attr(element_name, attr_name)
+        elements[element_name] ||= {}
+        elements[element_name][attr_name] ||= Attr.new(
+          attr_name, element_name, self
+        )
+      end
+
+      def schema
+        @schema ||= File.read(schema_path).split("\n")
+      end
+
+      private
+
+      def elements
+        @elements ||= {}
+      end
+
+      def schema_path
+        @schema_path ||= File.join(
+          cldr_requirement.common_path, 'dtd', 'ldml.dtd'
+        )
+      end
+    end
+
+  end
+end

data/lib/twitter_cldr/resources/cldr_locale.rb

@@ -0,0 +1,78 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'nokogiri'
+
+module TwitterCldr
+  module Resources
+
+    class CldrLocale
+      attr_reader :base_locale, :cldr_requirement
+
+      def initialize(base_locale, cldr_requirement)
+        @base_locale = base_locale
+        @cldr_requirement = cldr_requirement
+      end
+
+      def ancestors
+        @ancestors ||= [].tap do |ancestry|
+          locale = from_fs(base_locale)
+          ancestry << locale
+
+          loop do
+            cur = from_fs(ancestry.last)
+
+            if parents = parent_locales[cur]
+              ancestry << to_fs(parents)
+            elsif I18n::Locale::Tag.tag(cur).self_and_parents.count > 1
+              ancestry << I18n::Locale::Tag.tag(cur).self_and_parents.last.to_sym
+            else
+              parents = TwitterCldr::Shared::Locale
+                .parse(cur)
+                .permutations
+                .select do |loc|
+                  File.exist?(File.join(cldr_requirement.common_path, 'main', "#{loc}.xml"))
+                end
+
+              ancestry.concat(parents - [cur])
+              break
+            end
+          end
+
+          ancestry << "root"
+          ancestry.freeze
+        end
+      end
+
+      private
+
+      def to_fs(locale)
+        locale.to_s.gsub('_', '-').to_sym
+      end
+
+      def from_fs(locale)
+        locale.to_s.gsub('-', '_')
+      end
+
+      def parent_locales
+        @parent_locales ||= supplemental_data.xpath('//parentLocales/parentLocale').each_with_object({}) do |node, ret|
+          parent = node.attr('parent')
+          locales = node.attr('locales').split(' ')
+
+          locales.each do |locale|
+            ret[locale] = parent
+          end
+        end
+      end
+
+      def supplemental_data
+        @supplemental_data ||= Nokogiri.XML(
+          File.read(File.join(cldr_requirement.common_path, 'supplemental', 'supplementalData.xml'))
+        )
+      end
+    end
+
+  end
+end

data/lib/twitter_cldr/resources/list_formats_importer.rb

@@ -49,7 +49,7 @@ module TwitterCldr
         # locales, we fill in any missing parts at the minor risk of being slightly wrong
         # when formatting lists. In my opinion, it's far better to produce a slightly wrong
         # string than to error or produce an entirely empty string.
-        data = requirements[:cldr].merge_each_ancestor(locale) do |ancestor_locale|
+        data = requirements[:cldr].build_data(locale) do |ancestor_locale|
           ListFormats.new(ancestor_locale, requirements[:cldr]).to_h
         end
 

data/lib/twitter_cldr/resources/loader.rb

@@ -3,6 +3,8 @@
 # Copyright 2012 Twitter, Inc
 # http://www.apache.org/licenses/LICENSE-2.0
 
+require 'yaml'
+
 module TwitterCldr
   module Resources
 
@@ -10,7 +12,18 @@ module TwitterCldr
 
     class Loader
 
+      class << self
+        def load_yaml(yaml, permitted_classes: [])
+          if Psych::VERSION >= '4'
+            YAML.safe_load(yaml, permitted_classes: permitted_classes)
+          else
+            YAML.safe_load(yaml, permitted_classes)
+          end
+        end
+      end
+
       VALID_EXTS = %w(.yml .dump).freeze
+      PERMITTED_YAML_CLASSES = [Range, Regexp, Symbol, Time].freeze
 
       def get_resource(*path)
         resources_cache[resource_file_path(path)]
@@ -105,7 +118,7 @@ module TwitterCldr
       end
 
       def load_yaml_resource(path, merge_custom = true)
-        base = YAML.load(read_resource_file(path))
+        base = load_yaml(read_resource_file(path), permitted_classes: PERMITTED_YAML_CLASSES)
         custom_path = File.join("custom", path)
 
         if merge_custom && custom_resource_exists?(custom_path) && !TwitterCldr.disable_custom_locale_resources
@@ -115,6 +128,10 @@ module TwitterCldr
         base
       end
 
+      def load_yaml(yaml, permitted_classes: [])
+        self.class.load_yaml(yaml, permitted_classes: permitted_classes)
+      end
+
       def load_marshalled_resource(path, _merge_custom = :unused)
         Marshal.load(read_resource_file(path))
       end

data/lib/twitter_cldr/resources/number_formats_importer.rb

@@ -36,7 +36,7 @@ module TwitterCldr
       end
 
       def import_locale(locale)
-        data = requirements[:cldr].merge_each_ancestor(locale) do |ancestor_locale|
+        data = requirements[:cldr].build_data(locale) do |ancestor_locale|
           NumberFormats.new(ancestor_locale, requirements[:cldr]).to_h
         end
 

data/lib/twitter_cldr/resources/readme_renderer.rb

@@ -4,6 +4,7 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 
 require 'erb'
+require 'yaml'
 
 module TwitterCldr
   module Resources
@@ -79,8 +80,6 @@ module TwitterCldr
         end
       end
 
-      private
-
       def line_num_from_stack_trace(trace)
         trace[0].split(":")[1].to_i  # kind of a hack...
       end
@@ -101,6 +100,11 @@ module TwitterCldr
           ret
         end
       end
+
+      def tested_ruby_versions
+        workflow = YAML.load_file(File.join(*%w(.github workflows unit_tests.yml)))
+        workflow["jobs"]["build"]["strategy"]["matrix"]["ruby-version"].join(", ")
+      end
     end
 
   end

data/lib/twitter_cldr/resources/requirements/cldr_requirement.rb

@@ -42,15 +42,6 @@ module TwitterCldr
           File.join(source_path, 'common')
         end
 
-        def merge_each_ancestor(locale, merge_root: true)
-          locales = locale_chain(locale)
-          locales << :root if merge_root
-
-          locales.inject({}) do |result, ancestor_locale|
-            deep_merge(yield(ancestor_locale), result)
-          end
-        end
-
         def alt?(node)
           !node.attribute('alt').nil?
         end
@@ -62,49 +53,26 @@ module TwitterCldr
           )
         end
 
-        private
-
-        def locale_chain(locale)
-          locale = from_fs(locale)
-          ancestry = [locale]
-
-          loop do
-            cur = from_fs(ancestry.last)
-
-            if parents = parent_locales[cur]
-              ancestry << to_fs(parents)
-            elsif I18n::Locale::Tag.tag(cur).self_and_parents.count > 1
-              ancestry << I18n::Locale::Tag.tag(cur).self_and_parents.last.to_sym
-            else
-              parents = TwitterCldr::Shared::Locale
-                .parse(cur)
-                .permutations
-                .select do |loc|
-                  File.exist?(File.join(common_path, 'main', "#{loc}.xml"))
-                end
-
-              ancestry += parents - [cur]
-              break
-            end
-          end
-
-          ancestry
+        def build_data(locale, &block)
+          CldrDataBuilder.new(locale_for(locale)).merge_each_ancestor(&block)
         end
 
-        def to_fs(locale)
-          locale.to_s.gsub('_', '-').to_sym
+        def docset(path, root_locale)
+          CldrDocumentSet.new(path, locale_for(root_locale), self)
         end
 
-        def from_fs(locale)
-          locale.to_s.gsub('-', '_')
+        def dtd
+          @dtd ||= CldrDTD.new(self)
         end
 
-        DEEP_MERGER = proc do |key, v1, v2|
-          Hash === v1 && Hash === v2 ? v1.merge(v2, &DEEP_MERGER) : (v2 || v1)
+        private
+
+        def locale_for(locale)
+          locales[locale] ||= CldrLocale.new(locale, self)
         end
 
-        def deep_merge(h1, h2)
-          h1.merge(h2, &DEEP_MERGER)
+        def locales
+          @locales ||= {}
         end
 
         def cldr_url
@@ -134,23 +102,6 @@ module TwitterCldr
             end
           end
         end
-
-        def parent_locales
-          @parent_locales ||= supplemental_data.xpath('//parentLocales/parentLocale').each_with_object({}) do |node, ret|
-            parent = node.attr('parent')
-            locales = node.attr('locales').split(' ')
-
-            locales.each do |locale|
-              ret[locale] = parent
-            end
-          end
-        end
-
-        def supplemental_data
-          @supplemental_data ||= Nokogiri.XML(
-            File.read(File.join(common_path, 'supplemental', 'supplementalData.xml'))
-          )
-        end
       end
 
     end

data/lib/twitter_cldr/resources/territories_importer.rb

@@ -26,7 +26,7 @@ module TwitterCldr
       end
 
       def import_locale(locale)
-        data = requirements[:cldr].merge_each_ancestor(locale) do |ancestor_locale|
+        data = requirements[:cldr].build_data(locale) do |ancestor_locale|
           TerritoryData.new(ancestor_locale, requirements[:cldr]).to_h
         end
 

data/lib/twitter_cldr/resources/timezones_importer.rb

@@ -38,7 +38,7 @@ module TwitterCldr
       end
 
       def import_locale(locale)
-        data = requirements[:cldr].merge_each_ancestor(locale) do |ancestor_locale|
+        data = requirements[:cldr].build_data(locale) do |ancestor_locale|
           TimezoneData.new(ancestor_locale, requirements[:cldr]).to_h
         end
 

data/lib/twitter_cldr/resources/unicode_property_aliases_importer.rb

@@ -17,6 +17,8 @@ module TwitterCldr
       private
 
       def execute
+        FileUtils.mkdir_p(output_path)
+
         File.write(
           File.join(output_path, 'property_value_aliases.yml'),
           YAML.dump(parse_property_value_aliases)

data/lib/twitter_cldr/resources/units_importer.rb

@@ -36,7 +36,7 @@ module TwitterCldr
       end
 
       def import_locale(locale)
-        data = requirements[:cldr].merge_each_ancestor(locale) do |ancestor_locale|
+        data = requirements[:cldr].build_data(locale) do |ancestor_locale|
           Units.new(ancestor_locale, requirements[:cldr]).to_h
         end
 

data/lib/twitter_cldr/resources.rb

@@ -10,6 +10,10 @@ module TwitterCldr
     autoload :BidiTestImporter,               'twitter_cldr/resources/bidi_test_importer'
     autoload :CalendarsImporter,              'twitter_cldr/resources/calendars_importer'
     autoload :CasefolderClassGenerator,       'twitter_cldr/resources/casefolder_class_generator'
+    autoload :CldrDataBuilder,                'twitter_cldr/resources/cldr_data_builder'
+    autoload :CldrDocumentSet,                'twitter_cldr/resources/cldr_document_set'
+    autoload :CldrDTD,                        'twitter_cldr/resources/cldr_dtd'
+    autoload :CldrLocale,                     'twitter_cldr/resources/cldr_locale'
     autoload :CollationTestsImporter,         'twitter_cldr/resources/collation_tests_importer'
     autoload :CollationTriesImporter,         'twitter_cldr/resources/collation_tries_importer'
     autoload :CurrencySymbolsImporter,        'twitter_cldr/resources/currency_symbols_importer'
@@ -25,6 +29,7 @@ module TwitterCldr
     autoload :ParentLocalesImporter,          'twitter_cldr/resources/parent_locales_importer'
     autoload :PostalCodesImporter,            'twitter_cldr/resources/postal_codes_importer'
     autoload :Properties,                     'twitter_cldr/resources/properties'
+    autoload :ResourceLoadError,              'twitter_cldr/resources/loader'
     autoload :SegmentDictionariesImporter,    'twitter_cldr/resources/segment_dictionaries_importer'
     autoload :SegmentRulesImporter,           'twitter_cldr/resources/segment_rules_importer'
     autoload :SegmentTestsImporter,           'twitter_cldr/resources/segment_tests_importer'
@@ -54,6 +59,8 @@ module TwitterCldr
           AliasesImporter,
           Bcp47TimezoneMetadataImporter,
           BidiTestImporter,
+          ValidityDataImporter,
+          UnicodePropertyAliasesImporter,
           CalendarsImporter,
           CasefolderClassGenerator,
           CollationTestsImporter,
@@ -79,9 +86,7 @@ module TwitterCldr
           TransformTestsImporter,
           TransformsImporter,
           UnicodeDataImporter,
-          UnicodePropertyAliasesImporter,
-          UnitsImporter,
-          ValidityDataImporter,
+          UnitsImporter
         ]
       end
 

data/lib/twitter_cldr/shared/calendar.rb

@@ -7,7 +7,7 @@ module TwitterCldr
   module Shared
     class Calendar
 
-      DEFAULT_FORMAT = :'stand-alone'
+      DEFAULT_FORMAT = :format
 
       NAMES_FORMS = [:wide, :narrow, :short, :abbreviated]
       ERAS_NAMES_FORMS = [:abbr, :name]

data/lib/twitter_cldr/version.rb

@@ -4,5 +4,5 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 
 module TwitterCldr
-  VERSION = '6.9.0'
+  VERSION = '6.11.2'
 end

data/resources/locales/ar/calendars.yml

@@ -129,7 +129,8 @@
           0: "قبل الحقبة الحالية"
           1: "بعد الميلاد"
         :narrow: 
-          0: ""
+          0: BCE
+          1: CE
       :fields: 
         :day: يوم
         :day-narrow: يوم

data/resources/locales/az/calendars.yml

@@ -126,7 +126,8 @@
           0: "bizim eradan əvvəl"
           1: "bizim era"
         :narrow: 
-          0: ""
+          0: BCE
+          1: CE
       :fields: 
         :day: Gün
         :day-narrow: Gün

data/resources/locales/be/calendars.yml

@@ -127,7 +127,8 @@
           0: "да нашай эры"
           1: "нашай эры"
         :narrow: 
-          0: ""
+          0: BCE
+          1: CE
       :fields: 
         :day: дзень
         :day-narrow: д.

data/resources/locales/bg/calendars.yml

@@ -134,7 +134,8 @@
           0: "преди новата ера"
           1: "след новата ера"
         :narrow: 
-          0: ""
+          0: BCE
+          1: CE
       :fields: 
         :day: ден
         :day-narrow: д

data/resources/locales/bn/calendars.yml

@@ -129,7 +129,8 @@
           0: খ্রিষ্টপূর্বাব্দ
           1: খ্রিষ্টাব্দ
         :narrow: 
-          0: ""
+          0: BCE
+          1: CE
       :fields: 
         :day: দিন
         :day-narrow: দিন