twitter_cldr 5.2.0 → 5.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b616d55c343da67733837c0f1549329d895ea3758011b4c5dd8c90c3c631f53a
-  data.tar.gz: 3717867c2412adcc7a95ff1dfbe032e1754ade3cc005b4445d1f8ff644048b06
+  metadata.gz: 2783ec225d4f260deb8038237125dbc97b78840e951cbff630f3da502e31f41d
+  data.tar.gz: 751cc8931ee11db35a533584a1b9f3d8a946e9505641d33caad6a5a5dbc6e866
 SHA512:
-  metadata.gz: f82323e912a622930f192a2ffe8b742ce1378feff847a082e04b7ea0feb5df215faa183861e31a35e69bc966c6222d0182689cdc5264de69263acbc60acce8ff
-  data.tar.gz: dc11c5d5e3ab6cc0f2cf3a6073686f7167ac6621252a0b6bf4c7f0b1eb53ae7134507aa0d66e5c0c57e7abc9b689a6438032b4982abe766b9c8550b6df0139b8
+  metadata.gz: 7cdc1ec2718ac86b0645813fc0bdfe13b1c2bee075ca90e13231a3d566d915e7ef9bcdf6d69f8d0a7cbcdaf6868eae5c27bfb4b32b52efbd7dcf1adabc20c39d
+  data.tar.gz: 20b7e24ec990cc00fb77d60d62c75247108748629c5478a0a315d96e04a76c443f8abc2a4df21b7c4093ad7cc46ee81c4276ca15f97c8698073b7ac9bca98c65

data/Gemfile

@@ -23,10 +23,6 @@ group :development do
   gem 'ruby-cldr', github: 'camertron/ruby-cldr', branch: 'mapzones' # 'svenfuchs/ruby-cldr'
   gem 'i18n'
   gem 'cldr-plurals', '~> 1.0'
-
-  gem 'rest-client', '~> 1.8'
-
-  gem 'parallel'
 end
 
 group :test do

data/Rakefile

@@ -57,16 +57,22 @@ task :update do
 end
 
 task :add_locale, :locale do |_, args|
+  locales = [args[:locale]] + args.extras
+
   File.write(
     TwitterCldr::SUPPORTED_LOCALES_FILE,
     YAML.dump(
-      (TwitterCldr::SUPPORTED_LOCALES + [args[:locale]]).map(&:to_sym).uniq.sort
+      (TwitterCldr::SUPPORTED_LOCALES + locales).map(&:to_sym).uniq.sort
     )
   )
 
   klasses = TwitterCldr::Resources.locale_based_importer_classes_for_ruby_engine
-  instances = klasses.map { |klass| klass.new(locales: [args[:locale]]) }
-  TwitterCldr::Resources::ImportResolver.new(instances).import
+  instances = klasses.map { |klass| klass.new(locales: locales) }
+  resolver = TwitterCldr::Resources::ImportResolver.new(
+    instances, allow_missing_dependencies: true
+  )
+
+  resolver.import
 end
 
 # add_locale and update_locale do the same thing
@@ -160,6 +166,16 @@ namespace :update do
     TwitterCldr::Resources::SegmentRulesImporter.new.import
   end
 
+  desc 'Import segmentation dictionaries'
+  task :segment_dictionaries do
+    TwitterCldr::Resources::SegmentDictionariesImporter.new.import
+  end
+
+  desc 'Import segment tests'
+  task :segment_tests do
+    TwitterCldr::Resources::SegmentTestsImporter.new.import
+  end
+
   desc 'Import (generate) bidi tests (should be executed using JRuby 1.7 in 1.9 mode)'
   task :bidi_tests do
     TwitterCldr::Resources::BidiTestImporter.new.import
@@ -200,11 +216,6 @@ namespace :update do
     TwitterCldr::Resources::TransformTestsImporter.new.import
   end
 
-  desc 'Import segment tests'
-  task :segment_tests do
-    TwitterCldr::Resources::SegmentTestsImporter.new.import
-  end
-
   desc 'Import hyphenation dictionaries'
   task :hyphenation_dictionaries do
     TwitterCldr::Resources::HyphenationImporter.new.import

data/lib/twitter_cldr/normalization.rb

@@ -14,12 +14,25 @@ module TwitterCldr
     class << self
 
       def normalize(string, options = {})
-        form = options.fetch(:using, DEFAULT_NORMALIZER).to_s.downcase.to_sym
+        validate_form(form = extract_form_from(options))
+        Eprun.normalize(string, form)
+      end
+
+      def normalized?(string, options = {})
+        validate_form(form = extract_form_from(options))
+        Eprun.normalized?(string, form)
+      end
+
+      private
+
+      def extract_form_from(options)
+        options.fetch(:using, DEFAULT_NORMALIZER).to_s.downcase.to_sym
+      end
 
-        if VALID_NORMALIZERS.include?(form)
-          Eprun.normalize(string, form)
-        else
-          raise ArgumentError.new("#{form.inspect} is not a valid normalizer (valid normalizers are #{VALID_NORMALIZERS.join(', ')})")
+      def validate_form(form)
+        unless VALID_NORMALIZERS.include?(form)
+          raise ArgumentError.new("#{form.inspect} is not a valid normalizer "\
+            "(valid normalizers are #{VALID_NORMALIZERS.join(', ')})")
         end
       end
 

data/lib/twitter_cldr/resources.rb

@@ -22,12 +22,13 @@ module TwitterCldr
     autoload :NumberFormatsImporter,          'twitter_cldr/resources/number_formats_importer'
     autoload :PostalCodesImporter,            'twitter_cldr/resources/postal_codes_importer'
     autoload :Properties,                     'twitter_cldr/resources/properties'
+    autoload :SegmentDictionariesImporter,    'twitter_cldr/resources/segment_dictionaries_importer'
     autoload :SegmentRulesImporter,           'twitter_cldr/resources/segment_rules_importer'
+    autoload :SegmentTestsImporter,           'twitter_cldr/resources/segment_tests_importer'
     autoload :RbnfTestImporter,               'twitter_cldr/resources/rbnf_test_importer'
     autoload :ReadmeRenderer,                 'twitter_cldr/resources/readme_renderer'
     autoload :RegexpAstGenerator,             'twitter_cldr/resources/regexp_ast_generator'
     autoload :Requirements,                   'twitter_cldr/resources/requirements'
-    autoload :SegmentTestsImporter,           'twitter_cldr/resources/segment_tests_importer'
     autoload :TailoringImporter,              'twitter_cldr/resources/tailoring_importer'
     autoload :TerritoriesImporter,            'twitter_cldr/resources/territories_importer'
     autoload :TimezonesImporter,              'twitter_cldr/resources/timezones_importer'
@@ -60,6 +61,7 @@ module TwitterCldr
           NumberFormatsImporter,
           PostalCodesImporter,
           RbnfTestImporter,
+          SegmentDictionariesImporter,
           SegmentRulesImporter,
           SegmentTestsImporter,
           TailoringImporter,

data/lib/twitter_cldr/resources/import_resolver.rb

@@ -5,10 +5,11 @@ module TwitterCldr
     class ImportResolver
       include TSort
 
-      attr_reader :importers
+      attr_reader :importers, :options
 
-      def initialize(importers = Resources.importer_classes_for_ruby_engine)
+      def initialize(importers = Resources.importer_classes_for_ruby_engine, options = {})
         @importers = importers
+        @options = options
       end
 
       def import
@@ -28,7 +29,12 @@ module TwitterCldr
 
       def tsort_each_child(instance, &block)
         deps_for(instance).map do |dep_class|
-          yield instances.find { |ins| ins.class == dep_class }
+          dep = instances.find { |ins| ins.class == dep_class }
+          yield dep if dep
+
+          unless options[:allow_missing_dependencies]
+            raise "Could not find dependency #{dep_class.name}"
+          end
         end
       end
 
@@ -39,6 +45,8 @@ module TwitterCldr
       end
 
       def check_unmet_instance_deps(instance)
+        return if options[:allow_missing_dependencies]
+
         unmet_deps = unmet_deps_for(instance)
 
         unless unmet_deps.empty?

data/lib/twitter_cldr/resources/loader.rb

@@ -10,6 +10,8 @@ module TwitterCldr
 
     class Loader
 
+      VALID_EXTS = %w(.yml .dump).freeze
+
       def get_resource(*path)
         resources_cache[resource_file_path(path)]
       end
@@ -75,7 +77,7 @@ module TwitterCldr
 
       def resource_file_path(path)
         file = File.join(*path.map(&:to_s))
-        file << '.yml' unless file.end_with?('.yml')
+        file << '.yml' unless VALID_EXTS.include?(File.extname(file))
         file
       end
 
@@ -92,6 +94,17 @@ module TwitterCldr
       end
 
       def load_resource(path, merge_custom = true)
+        case File.extname(path)
+          when '.yml'
+            load_yaml_resource(path, merge_custom)
+          when '.dump'
+            load_marshalled_resource(path, merge_custom)
+          else
+            load_raw_resource(path, merge_custom)
+        end
+      end
+
+      def load_yaml_resource(path, merge_custom = true)
         base = YAML.load(read_resource_file(path))
         custom_path = File.join("custom", path)
 
@@ -102,6 +115,14 @@ module TwitterCldr
         base
       end
 
+      def load_marshalled_resource(path, _merge_custom = :unused)
+        Marshal.load(read_resource_file(path))
+      end
+
+      def load_raw_resource(path, _merge_custom = :unused)
+        read_resource_file(path)
+      end
+
       def custom_resource_exists?(custom_path)
         File.exist?(
           File.join(TwitterCldr::RESOURCES_DIR, custom_path)

data/lib/twitter_cldr/resources/locales_resources_importer.rb

@@ -56,13 +56,6 @@ module TwitterCldr
         params.fetch(:output_path)
       end
 
-      def move_segments_root_file
-        old_file_path = File.join(output_path, *%w(shared segments_root.yml))
-        new_file_path = File.join(output_path, *%w(shared segments segments_root.yml))
-        FileUtils.mkdir_p(File.dirname(new_file_path))
-        FileUtils.move(old_file_path, new_file_path)
-      end
-
       def import_components
         locales = Set.new
 
@@ -100,8 +93,6 @@ module TwitterCldr
         Cldr::Export.export(export_args) do |component, locale, path|
           deep_symbolize(path)
         end
-
-        move_segments_root_file
       end
 
       def components_for(locale)

data/lib/twitter_cldr/resources/postal_codes_importer.rb

@@ -3,8 +3,8 @@
 # Copyright 2012 Twitter, Inc
 # http://www.apache.org/licenses/LICENSE-2.0
 
-require 'rest-client'
 require 'json'
+require 'open-uri'
 require 'set'
 require 'yaml'
 
@@ -21,38 +21,36 @@ module TwitterCldr
       private
 
       def execute
-        File.open(File.join(output_path, 'postal_codes.yml'), 'w') do |output|
-          output.write(YAML.dump(load))
-        end
+        data = YAML.dump(fetch_data)
+        File.write(File.join(output_path, 'postal_codes.yml'), data)
+        puts
       end
 
       def output_path
         params.fetch(:output_path)
       end
 
-      def load
+      def fetch_data
         territories = Set.new
 
         each_territory.each_with_object({}) do |territory, ret|
-          next unless regex = get_regex_for(territory)
-
-          ret[territory] = {
-            regex: Regexp.compile(regex),
-            ast: TwitterCldr::Utils::RegexpAst.dump(
-              RegexpAstGenerator.generate(regex)
-            )
-          }
+          if regex = get_regex_for(territory)
+            ret[territory] = {
+              regex: Regexp.compile(regex),
+              ast: TwitterCldr::Utils::RegexpAst.dump(
+                RegexpAstGenerator.generate(regex)
+              )
+            }
+          end
 
           territories.add(territory)
           STDOUT.write("\rImported postal codes for #{territory}, #{territories.size} of #{territory_count} total")
         end
-
-        puts
       end
 
       def get_regex_for(territory)
-        result = RestClient.get("#{BASE_URL}#{territory.to_s.upcase}")
-        data = JSON.parse(result.body)
+        result = URI.open("#{BASE_URL}#{territory.to_s.upcase}").read
+        data = JSON.parse(result)
         data['zip']
       end
 
@@ -61,12 +59,10 @@ module TwitterCldr
       end
 
       def each_territory
-        if block_given?
-          TwitterCldr::Shared::Territories.all.each_pair do |territory, _|
-            yield territory
-          end
-        else
-          to_enum(__method__)
+        return to_enum(__method__) unless block_given?
+
+        TwitterCldr::Shared::Territories.all.each_pair do |territory, _|
+          yield territory
         end
       end
 

data/lib/twitter_cldr/resources/segment_dictionaries_importer.rb

@@ -0,0 +1,75 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'fileutils'
+require 'open-uri'
+
+module TwitterCldr
+  module Resources
+    class SegmentDictionariesImporter < Importer
+
+      URL_TEMPLATE = 'https://raw.githubusercontent.com/unicode-org/icu/%{icu_version}/%{path}'
+
+      DICTIONARY_FILES = [
+        'icu4c/source/data/brkitr/dictionaries/burmesedict.txt',
+        'icu4c/source/data/brkitr/dictionaries/cjdict.txt',
+        'icu4c/source/data/brkitr/dictionaries/khmerdict.txt',
+        'icu4c/source/data/brkitr/dictionaries/laodict.txt',
+        'icu4c/source/data/brkitr/dictionaries/thaidict.txt'
+      ]
+
+      output_path File.join(*%w(shared segments dictionaries))
+      ruby_engine :mri
+
+      def execute
+        FileUtils.mkdir_p(output_path)
+
+        DICTIONARY_FILES.each do |test_file|
+          import_dictionary_file(test_file)
+        end
+      end
+
+      private
+
+      def import_dictionary_file(dictionary_file)
+        source_url = url_for(dictionary_file)
+        source = open(source_url).read
+        lines = source.split("\n")
+        trie = TwitterCldr::Utils::Trie.new
+        space_regexp = TwitterCldr::Shared::UnicodeRegex.compile('\A[[:Z:][:C:]]+').to_regexp
+
+        lines.each do |line|
+          line.sub!(space_regexp, '')
+          next if line.start_with?('#')
+
+          characters, frequency = line.split("\t")
+          frequency = frequency ? frequency.to_i : 0
+
+          trie.add(characters.unpack('U*'), frequency)
+        end
+
+        output_path = output_path_for(dictionary_file)
+        File.write(output_path, Marshal.dump(trie))
+      end
+
+      def url_for(dictionary_file)
+        URL_TEMPLATE % {
+          icu_version: "release-#{Versions.icu_version.gsub('.', '-')}",
+          path: dictionary_file
+        }
+      end
+
+      def output_path_for(dictionary_file)
+        file = File.basename(dictionary_file).chomp(File.extname(dictionary_file))
+        File.join(output_path, "#{file}.dump")
+      end
+
+      def output_path
+        params.fetch(:output_path)
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/resources/segment_tests_importer.rb

@@ -9,47 +9,164 @@ module TwitterCldr
   module Resources
     class SegmentTestsImporter < Importer
 
-      TEST_FILES = [
+      CONFORMANCE_FILES = [
         'ucd/auxiliary/WordBreakTest.txt',
         'ucd/auxiliary/SentenceBreakTest.txt',
         'ucd/auxiliary/GraphemeBreakTest.txt',
         'ucd/auxiliary/LineBreakTest.txt'
       ]
 
-      requirement :unicode, Versions.unicode_version, TEST_FILES
+      DICTIONARY_BREAK_SAMPLES = {
+        # Chinese
+        zh: '無畏號航空母艦是一艘隸屬於美國海軍的航空母艦，為艾塞克斯級航空母艦的三號艦。'\
+            '無畏號於1941年開始建造，1943年下水服役，開始參與太平洋戰爭。戰後無畏號退役封存，'\
+            '在韓戰後開始進行SCB-27C改建，又在期間重編為攻擊航母，於1954年在大西洋艦隊重新服役。'\
+            '稍後無畏號又進行SCB-125現代化改建，增設斜角飛行甲板。1962年無畏號重編為反潛航母，'\
+            '舷號改為CVS-11，繼續留在大西洋及地中海執勤。稍後無畏號參與美國的太空計畫，'\
+            '分別擔任水星-宇宙神7號及雙子座3號的救援船。1966年至1969年，無畏號曾三次前往西太平洋，'\
+            '參與越戰。無畏號在1974年退役，並一度預備出售拆解；但在民間組織努力下，'\
+            '海軍在1981年將無畏號捐贈到紐約作博物館艦。1986年，無畏號獲評為美國國家歷史地標。',
+
+        ko: '파일은 이용자가 공용 또는 위키백과 한국어판에 업로드하여 라이선스에 따라 사용 가능한 형태로 제공됩니다. '\
+            '업로드된 파일은 간단한 조작으로 페이지에 삽입할 수 있습니다. 업로드는 신규 이용자를 제외한 등록 이용자라면 '\
+            '가능합니다. 파일을 업로드하기 전에 다음 문단의 업로드를 할 수 없는 파일을 반드시 읽어 보시기 바랍니다. '\
+            '공용 이용 방법 및 업로드에 대해서는 Commons:초보자 길라잡이/업로드를 읽어 보시기 바랍니다. 업로드하는 '\
+            '페이지는 위키백과:파일 올리기를 참조하십시오. 파일의 라이선스가 삽입되는 위키백과의 문서와는 별도로 '\
+            '개별적으로 설정해야 합니다. 파일을 업로드할 때 적절한 라이선스를 선택하고 반드시 표시하십시오.',
+
+        # Thai
+        th: 'ธงไชย แมคอินไตย์ ชื่อเล่น เบิร์ด (เกิด 8 ธันวาคม พ.ศ. 2501) เป็นนักร้อง นักแสดงชาวไทย '\
+            'ได้รับขนานนามว่าเป็น "ซูเปอร์สตาร์เมืองไทย" โดยคนไทยรู้จักกันดี เรียกกันว่า : พี่เบิร์ด '\
+            'แรกเข้าวงการบันเทิงเป็นนักแสดงสมทบ ต่อมาได้รับบทพระเอก โดยภาพยนตร์ที่สร้างชื่อเสียงให้กับเขาที่สุดเรื่อง '\
+            'ด้วยรักคือรัก ส่วนละครที่สร้างชื่อเสียงที่สุดของเขาคือบท "โกโบริ" ในละครคู่กรรม '\
+            'ด้านวงการเพลงซึ่งเป็นอาชีพหลักเขาเริ่มต้นจากการประกวดร้องเพลงของสยามกลการ '\
+            'ต่อมาเป็นนักร้องในสังกัดบริษัท จีเอ็มเอ็ม แกรมมี่ จำกัด (มหาชน) ซึ่งประสบความสำเร็จสูงสุดของประเทศไทย'\
+            'มียอดจำหน่ายอยู่ในระดับแนวหน้าของทวีปเอเชียยอดรวมกว่า 25 ล้านชุด',
+
+        # Khmer
+        km: 'វីគីភីឌា (អង់គ្លេស ៖ Wikipedia) ជាសព្វវចនាធិប្បាយសេរីច្រើនភាសានៅលើអ៊ីនធឺណិត '\
+            'ដែលមនុស្សគ្រប់គ្នាអាចអាននិងធ្វើឱ្យមាតិកាទាន់សម័យបន្ថែមទៀត '\
+            'ធ្វើឱ្យវីគីភីឌាសព្វវចនាធិប្បាយបានក្លាយទៅជាការកែប្រែ '\
+            'ការប្រមូលនិងការអភិរក្សរាប់រយរាប់ពាន់នាក់នៃអ្នកស្ម័គ្រចិត្តនៅជុំវិញពិភពលោក '\
+            'តាមរយៈកម្មវិធីដែលគេហៅថាមេឌាវិគី ។ វីគីភីឌាចាប់ផ្តើមនៅថ្ងៃទី ១៥ មករា ឆ្នាំ ២០០១ '\
+            'ដោយចាប់ផ្តើមគម្រោងពីឈ្មោះសព្វវចនាធិប្បាយណូ៉ភីឌាដែលសរសេរដោយហ្ស៊ីម្ម៊ី '\
+            'វេល្ស និងឡែរ្រី សែងក័រ ។ នៅបច្ចុប្បន្ននេះ វីគីភីឌាមានទាំង់អស់ ២៩៣ ភាសា[៤] ដោយវីគីភីឌាភាសាខ្មែរមាន '\
+            '៧៨៩៨ អត្ថបទ ។ មានវីគីភីឌាច្រើនជាង ៥០ ភាសាដែលមានអត្ថបទច្រើនជាង ១០០.០០០ អត្ថបទ ។ '\
+            'វីគីភីឌាភាសាអាល្លឺម៉ងត្រូវបានគេចែកចាយនៅក្នុងទ្រង់ទ្រាយឌីវីឌី-រ៉ូម ។',
+
+        # Lao
+        lo: 'ວິກິພີເດຍ (ອັງກິດ: Wikipedia) ເປັນສາລະນຸກົມເນື້ອຫາເສລີຫຼາຍພາສາໃນເວັບໄຊ້ '\
+            'ເຊິ່ງໄດ້ຮັບການສະໜັບສະໜຸນຈາກມູນລະນິທິວິກິພີເດຍ ອົງກອນບໍ່ສະແຫວງຫາຜົນກຳໄລ ເນື້ອຫາກວ່າ 35 ລ້ານບົດຄວາມ '\
+            '(ສະເພາະວິກິພີເດຍພາສາອັງກິດມີເນື້ອຫາກວ່າ 4.9 ລ້ານບົດຄວາມ) ເກີດຂຶ້ນຈາກການຮ່ວມຂຽນຂອງອາສາສະໝັກທົ່ວໂລກ '\
+            'ທຸກຄົນທີ່ສາມາດເຂົ້າເຖິງວິກິພີເດຍສາມາດຮ່ວມແກ້ໄຂເກືອບທຸກບົດຄວາມໄດ້ຢ່າງເສລີ ໂດຍມີຜູ້ຂຽນປະມານ 100,000ຄົນ '\
+            'ຈົນເຖິງເດືອນເມສາ ຄ.ສ. 2013 ວິກິພີເດຍມີ 286 ຮຸ່ນພາສາ ແລະ '\
+            'ໄດ້ກາຍມາເປັນງານອ້າງອິງທົ່ວໄປທີ່ໃກຍ່ທີ່ສຸດແລະໄດ້ຮັບຄວາມນິຍົມຫຼາຍທີ່ສຸດຢູ່ອິນເຕີເນັດ ຈົນຖືກຈັດເປັນເວັບໄຊ້ ອັນດັບທີ 6 '\
+            'ທີ່ມີຜູ້ເຂົ້າເບິ່ງຫຼາຍທີ່ສຸດໃນໂລກ ຕາມການຈັດອັນດັບຂອງອາເລັກຊ້າ ດ້ວຍຈຳນວນຜູ້ອ່ານກວ່າ 365 ລ້ານຄົນ '\
+            'ມີການປະເມີນວ່າວິກິພີເດຍມີການຄົ້ນຫາຂໍ້ມູນໃນວິກິພີເດຍກວ່າ 2,700 ລ້ານເທື່ອຕໍ່ເດືອນໃນສະຫະລັດ ອາເມຣິກາ',
+
+        # Burmese
+        my: 'ကိန်းဆိုသည်မှာ ရေတွက်ရန်နှင့် တိုင်းတာရန် အတွက် အသုံးပြုသော သင်္ချာဆိုင်ရာ အရာဝတ္ထုတစ်ခု '\
+            'ဖြစ်သည်။ သင်္ချာပညာတွင် ကိန်းဂဏန်းများ၏ အဓိပ္ပာယ်ဖွင့်ဆိုချက်ကို တဖြည်းဖြည်း ချဲ့ကားလာခဲ့သဖြင့် '\
+            'နှစ်ပေါင်းများစွာ ကြာသောအခါတွင် သုည၊ အနှုတ်ကိန်းများ (negative numbers)၊ ရာရှင်နယ်ကိန်း '\
+            '(rational number) ခေါ် အပိုင်းကိန်းများ၊ အီရာရှင်နယ်ကိန်း (irrational number) ခေါ် '\
+            'အပိုင်းကိန်းမဟုတ်သောကိန်းများ နှင့် ကွန်ပလက်စ်ကိန်း (complex number) ခေါ် ကိန်းရှုပ်များ စသည်ဖြင့် '\
+            'ပါဝင်လာကြသည်။ သင်္ချာဆိုင်ရာ တွက်ချက်မှုများ (mathematical operations) တွင် ဂဏန်းတစ်ခု '\
+            'သို့မဟုတ် တစ်ခုထက်ပိုသော ဂဏန်းများကို အဝင်ကိန်းအဖြစ် လက်ခံကြပြီး ဂဏန်းတစ်ခုကို အထွက်ကိန်း '\
+            'အဖြစ် ပြန်ထုတ်ပေးသည်။ ယူနရီ တွက်ချက်မှု (unary operation) ခေါ် တစ်လုံးသွင်းတွက်ချက်မှုတွင် '\
+            'ဂဏန်းတစ်ခုကို အဝင်ကိန်း အဖြစ် လက်ခံပြီး ဂဏန်းတစ်ခုကို အထွက်ကိန်း အဖြစ် ထုတ်ပေးသည်။ '
+      }.freeze
+
+      requirement :unicode, Versions.unicode_version, CONFORMANCE_FILES
+      requirement :icu, Versions.icu_version
       output_path 'shared/segments/tests'
-      ruby_engine :mri
+      ruby_engine :jruby
 
       def execute
-        TEST_FILES.each do |test_file|
-          import_test_file(test_file)
-        end
+        import_conformance_files
+        import_dictionary_break_tests
+        import_combined_dictionary_break_test
       end
 
       private
 
-      def import_test_file(test_file)
-        source_file = source_path_for(test_file)
+      def import_conformance_files
+        CONFORMANCE_FILES.each do |test_file|
+          import_conformance_file(test_file)
+        end
+      end
+
+      def import_conformance_file(conformance_file)
+        source_file = conformance_source_path_for(conformance_file)
         FileUtils.mkdir_p(File.dirname(source_file))
         result = UnicodeFileParser.parse_standard_file(source_file).map(&:first)
-        output_file = output_path_for(test_file)
+        output_file = conformance_output_path_for(conformance_file)
         FileUtils.mkdir_p(File.dirname(output_file))
         File.write(output_file, YAML.dump(result))
       end
 
-      def source_path_for(test_file)
-        requirements[:unicode].source_path_for(test_file)
+      def import_dictionary_break_tests
+        DICTIONARY_BREAK_SAMPLES.each do |locale, text_sample|
+          data = create_dictionary_break_test(locale.to_s, text_sample)
+          dump_dictionary_break_test(locale, data)
+        end
       end
 
-      def output_path_for(test_file)
-        file = underscore(File.basename(test_file).chomp(File.extname(test_file)))
+      def import_combined_dictionary_break_test
+        text_sample = DICTIONARY_BREAK_SAMPLES.values.join(' ')
+        data = create_dictionary_break_test('en', text_sample)
+        dump_dictionary_break_test('combined', data)
+      end
+
+      def create_dictionary_break_test(locale, text_sample)
+        done = break_iterator.const_get(:DONE)
+        iter = break_iterator.get_word_instance(ulocale_class.new(locale))
+        iter.set_text(text_sample)
+        start = iter.first
+        segments = []
+
+        until (stop = iter.next) == done
+          segments << text_sample[start...stop]
+          start = stop
+        end
+
+        {
+          locale: locale,
+          text: text_sample,
+          segments: segments
+        }
+      end
+
+      def dump_dictionary_break_test(name, data)
+        output_file = dictionary_test_output_path_for(name)
+        FileUtils.mkdir_p(File.dirname(output_file))
+        File.write(output_file, YAML.dump(data))
+      end
+
+      def conformance_source_path_for(conformance_file)
+        requirements[:unicode].source_path_for(conformance_file)
+      end
+
+      def conformance_output_path_for(conformance_file)
+        file = underscore(File.basename(conformance_file).chomp(File.extname(conformance_file)))
         File.join(params.fetch(:output_path), "#{file}.yml")
       end
 
+      def dictionary_test_output_path_for(locale)
+        File.join(params.fetch(:output_path), 'dictionary_tests', "#{locale}.yml")
+      end
+
       def underscore(str)
         str.gsub(/(.)([A-Z])/, '\1_\2').downcase
       end
 
+      def ulocale_class
+        @ulocale_class ||= requirements[:icu].get_class('com.ibm.icu.util.ULocale')
+      end
+
+      def break_iterator
+        @break_iterator ||= requirements[:icu].get_class('com.ibm.icu.text.BreakIterator')
+      end
+
     end
   end
 end