twitter_cldr 4.0.0 → 4.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 77d562387667ce48ca9f7fd86dfbfe300c387342
-  data.tar.gz: 3269c97b2f7b47a84264c1c4766cd6fcf2205a97
+  metadata.gz: a54546f0097f539bcbad9d1b42bdaeba9bbb32dc
+  data.tar.gz: a6b3d7233e0f0b7bce65dd5af721c3bc8faac763
 SHA512:
-  metadata.gz: 2ae5ee0153fd3c1c12aa558f7315d815af5f592a4924674adc5c8786fc87d402c5e7db65f62d48a1fbf7df596dc23f0997126eaee622c1af42f9e1e214763097
-  data.tar.gz: f003b9f2514a86b1f1aa1af1ab1ffaf04c233f200b38e1d63a9bc52a80fc4f802ba2ea741602562d577a04f05086ec0234c8ed93f5b01cb570560a13452bfd2f
+  metadata.gz: f0fb42535a4efd6f02165cc23b0498c79889d28dc549e3128735a022b468ca73f500e26884036cd6f15b705ece4474d7a12d9994119936f1b310b657c5a5966c
+  data.tar.gz: cf4e0fffe9b0811b73cd4b672cce2444c0dc30d548f8d7d35b030edcec4ba3774097c44dd9d31e6bcdbe9611e9f8ed7fdd61ed673e3f3fec4436176ed4bceec2

data/Gemfile

@@ -12,7 +12,7 @@ end
 group :development do
   gem 'nokogiri', "~> 1.5.9"
 
-  gem 'ruby-cldr', github: 'camertron/ruby-cldr', branch: 'fields'
+  gem 'ruby-cldr', github: 'camertron/ruby-cldr'
   gem 'i18n', '~> 0.6.11'
   gem 'cldr-plurals', '~> 1.0'
 
@@ -23,15 +23,16 @@ group :test do
   gem 'rspec', '~> 2.14.0'
   gem 'rr',    '~> 1.1.2'
 
-  gem 'rubyzip'
   gem 'term-ansicolor', '~> 1.3.0'        # 1.4 breaks ruby 1.9 support
   gem 'coveralls', require: false
   gem 'tins', '~> 1.6.0', require: false  # 1.7 breaks ruby 1.9 support
 
   gem 'simplecov'
   gem 'launchy'
+  gem 'addressable', '~> 2.4.0'           # 2.5 breaks ruby 1.9 support
 end
 
 group :development, :test do
+  gem 'rubyzip', '~> 1.0'
   gem 'benchmark-ips'
 end

data/README.md

@@ -18,7 +18,7 @@ require 'twitter_cldr'
 Get a list of all currently supported locales (these are all supported on twitter.com):
 
 ```ruby
-TwitterCldr.supported_locales             # [:af, :ar, :be, :bg, :bn, :ca, ... ]
+TwitterCldr.supported_locales             # [:af, :ar, :be, :bg, :bn, :bo, ... ]
 ```
 
 Determine if a locale is supported by TwitterCLDR:
@@ -554,7 +554,7 @@ postal_code.regexp  # /(\d{5})(?:[ \-](\d{4}))?/
 Get a sample of valid postal codes with the `#sample` method:
 
 ```ruby
-postal_code.sample(5)  # ["49037", "77863", "11631-2555", "09849-9066", "60383"]
+postal_code.sample(5)  # ["88435-2482", "19400", "34845-8011", "73815-5785", "43520-3829"]
 ```
 
 ### Phone Codes
@@ -1002,6 +1002,22 @@ TwitterCldr::Shared::YAML.dump(["hello", "world"])
 TwitterCldr::Shared::YAML.dump("hello, world") 
 ```
 
+## Adding New Locales
+
+TwitterCLDR doesn't support every locale available in the CLDR data set. If the library doesn't support the locale you need, feel free to add it and create a pull request. Adding (or updating) locales is easy. You'll need to run several rake tasks, one with MRI and another with JRuby. You'll also need an internet connection, since most of the tasks require downloading versions of CLDR, ICU, and various Unicode data files.
+
+Under MRI and then JRuby, run the `add_locale` rake task, passing the locale in the square brackets:
+
+```
+bundle exec rake add_locale[bo]
+```
+
+If you're using rbenv or rvm, try using the `add_locale.sh` script, which will install the required Ruby versions and run the rake tasks:
+
+```
+./script/add_locale.sh bo
+```
+
 ## About Twitter-specific Locales
 
 Twitter tries to always use BCP-47 language codes.  Data from the CLDR doesn't always match those codes however, so TwitterCLDR provides a `convert_locale` method to convert between the two.  All functionality throughout the entire gem defers to `convert_locale` before retrieving CLDR data.  `convert_locale` supports Twitter-supported BCP-47 language codes as well as CLDR locale codes, so you don't have to guess which one to use.  Here are a few examples:

data/Rakefile

@@ -52,183 +52,100 @@ namespace :spec do
 end
 
 task :update do
-  tasks = if RUBY_PLATFORM == 'java'
-    # these should be run using JRuby 1.7 in 1.9 mode, ICU4J v51.2, and CLDR v23.1 (v24 collation rules syntax is not supported yet)
-    [
-      "update:tailoring_data",  # per locale
-      "update:collation_tries", # per locale, must come after update:tailoring_data
-      "update:rbnf_tests",      # per locale
-      "update:transform_tests",
-      "update:bidi_tests"
-    ]
-  else
-    puts "You might also want to run this rake task using JRuby 1.7 (in 1.9 mode) to update collation data and RBNF tests."
-
-    [
-      "update:locales_resources",      # per locale (+ units resources using different CLDR and ruby-cldr, see LocalesResourcesImporter)
-      "update:unicode_data",
-      "update:unicode_properties",
-      "update:unicode_property_aliases",
-      "update:generate_casefolder",    # must come after unicode data
-      "update:postal_codes",
-      "update:phone_codes",
-      "update:language_codes",
-      "update:segment_exceptions",     # per locale
-      "update:segment_tests",
-      "update:readme",
-      "update:hyphenation"
-    ]
-  end
-
-  tasks.each do |task|
-    puts "Executing #{task}"
-    Rake::Task[task].invoke
-  end
+  klasses = TwitterCldr::Resources.importer_classes_for_ruby_engine
+  TwitterCldr::Resources::ImportResolver.new(klasses).import
 end
 
-# TODO: 'add_locale' task that creates a new directory and runs all necessary 'update' tasks (+ suggests to run those that depend on JRuby)
+task :add_locale, :locale do |_, args|
+  klasses = TwitterCldr::Resources.locale_based_importer_classes_for_ruby_engine
+  instances = klasses.map { |klass| klass.new(locales: [args[:locale]]) }
+  TwitterCldr::Resources::ImportResolver.new(instances).import
+end
 
-namespace :update do
-  ICU_JAR = './vendor/icu4j.jar'
+# add_locale and update_locale do the same thing
+task :update_locale, [:locale] => :add_locale
 
+namespace :update do
   desc 'Import locales resources'
-  task :locales_resources, :cldr_path do |_, args|
-    TwitterCldr::Resources::LocalesResourcesImporter.new(
-      args[:cldr_path] || './vendor/cldr',
-      './resources'
-    ).import
-  end
-
-  desc 'Import custom locales resources'
-  task :custom_locales_resources do
-    TwitterCldr::Resources::CustomLocalesResourcesImporter.new(
-      './resources/custom/locales'
-    ).import
+  task :locales_resources do
+    TwitterCldr::Resources::LocalesResourcesImporter.new.import
   end
 
   desc 'Import tailoring resources from CLDR data (should be executed using JRuby 1.7 in 1.9 mode)'
-  task :tailoring_data, :cldr_path, :icu4j_jar_path do |_, args|
-    TwitterCldr::Resources::TailoringImporter.new(
-      args[:cldr_path] || './vendor/cldr',
-      './resources/collation/tailoring',
-      args[:icu4j_jar_path] || ICU_JAR
-    ).import(TwitterCldr.supported_locales)
+  task :tailoring_data do
+    TwitterCldr::Resources::TailoringImporter.new.import
   end
 
   desc 'Import Unicode data resources'
-  task :unicode_data, :unicode_data_path do |_, args|
-    TwitterCldr::Resources::UnicodeDataImporter.new(
-      args[:unicode_data_path] || './vendor/unicode-data',
-      './resources/unicode_data'
-    ).import
+  task :unicode_data do
+    TwitterCldr::Resources::UnicodeDataImporter.new.import
   end
 
   desc 'Import Unicode property resources'
-  task :unicode_properties, :properties_path do |_, args|
-    TwitterCldr::Resources::Properties::PropertiesImporter.new(
-      args[:properties_path] || './vendor/unicode-data/properties',
-      './resources/unicode_data/properties'
-    ).import
+  task :unicode_properties do
+    TwitterCldr::Resources::property_importer_classes.each do |klass|
+      klass.new.import
+    end
   end
 
   desc 'Import unicode property value aliases'
-  task :unicode_property_aliases, :property_aliases_path do |_, args|
-    TwitterCldr::Resources::UnicodePropertyAliasesImporter.new(
-      args[:property_aliases_path] || './vendor/unicode-data',
-      './resources/unicode_data'
-    ).import
+  task :unicode_property_aliases do
+    TwitterCldr::Resources::UnicodePropertyAliasesImporter.new.import
   end
 
   desc 'Generate the casefolder class. Depends on unicode data'
-  task :generate_casefolder do
-    TwitterCldr::Resources::CasefolderClassGenerator.new(
-      './lib/twitter_cldr/resources/casefolder.rb.erb',
-      './lib/twitter_cldr/shared'
-    ).generate
+  task :casefolder do
+    TwitterCldr::Resources::CasefolderClassGenerator.new.import
   end
 
   desc 'Import postal codes resource'
   task :postal_codes do
-    TwitterCldr::Resources::PostalCodesImporter.new(
-      './resources/shared'
-    ).import
+    TwitterCldr::Resources::PostalCodesImporter.new.import
   end
 
   desc 'Import phone codes resource'
-  task :phone_codes, :cldr_path do |_, args|
-    TwitterCldr::Resources::PhoneCodesImporter.new(
-      args[:cldr_path] || './vendor/cldr',
-      './resources/shared'
-    ).import
+  task :phone_codes do
+    TwitterCldr::Resources::PhoneCodesImporter.new.import
   end
 
   desc 'Import language codes'
-  task :language_codes, :language_codes_data do |_, args|
-    TwitterCldr::Resources::LanguageCodesImporter.new(
-      args[:language_codes_data] || './vendor/language-codes',
-      './resources/shared'
-    ).import
+  task :language_codes do
+    TwitterCldr::Resources::LanguageCodesImporter.new.import
   end
 
   desc 'Update default and tailoring tries dumps (should be executed using JRuby 1.7 in 1.9 mode)'
   task :collation_tries do
-    TwitterCldr::Resources::CollationTriesDumper.update_dumps
-  end
-
-  desc 'Import normalization quick check data'
-  task :normalization_quick_check do
-    TwitterCldr::Resources::NormalizationQuickCheckImporter.new(
-      './vendor',
-      './resources/unicode_data'
-    ).import
+    TwitterCldr::Resources::CollationTriesImporter.new.import
   end
 
   desc 'Import (generate) bidi tests (should be executed using JRuby 1.7 in 1.9 mode)'
-  task :bidi_tests do |_, args|
-    TwitterCldr::Resources::BidiTestImporter.new(
-      './spec/bidi'
-    ).import
+  task :bidi_tests do
+    TwitterCldr::Resources::BidiTestImporter.new.import
   end
 
   desc 'Import (generate) rule-based number format tests (should be executed using JRuby 1.7 in 1.9 mode)'
-  task :rbnf_tests, :icu4j_jar_path do |_, args|
-    TwitterCldr::Resources::RbnfTestImporter.new(
-      './spec/formatters/numbers/rbnf/locales',
-      args[:icu4j_jar_path] || ICU_JAR
-    ).import(TwitterCldr.supported_locales)
+  task :rbnf_tests do
+    TwitterCldr::Resources::RbnfTestImporter.new.import
   end
 
   desc 'Import (generate) transformation tests (should be executed using JRuby 1.7 in 1.9 mode)'
-  task :transform_tests, :icu4j_jar_path do |_, args|
-    TwitterCldr::Resources::TransformTestImporter.new(
-      './spec/transforms/test_data.yml',
-      args[:icu4j_jar_path] || ICU_JAR
-    ).import
+  task :transform_tests do
+    TwitterCldr::Resources::TransformTestImporter.new.import
   end
 
   desc 'Import segment exceptions'
   task :segment_exceptions do
-    TwitterCldr::Resources::Uli::SegmentExceptionsImporter.new(
-      './vendor/uli/segments',
-      './resources/uli/segments'
-    ).import([:de, :en, :es, :fr, :it, :pt, :ru])  # only locales ULI supports at the moment
+    TwitterCldr::Resources::Uli::SegmentExceptionsImporter.new.import
   end
 
   desc 'Import segment tests'
   task :segment_tests do
-    TwitterCldr::Resources::SegmentTestsImporter.new(
-      './vendor/unicode-data/segments',
-      './resources/shared/segments/tests'
-    ).import
+    TwitterCldr::Resources::SegmentTestsImporter.new.import
   end
 
   desc 'Import hyphenation dictionaries'
   task :hyphenation_dictionaries do
-    TwitterCldr::Resources::HyphenationImporter.new(
-      './vendor/hyphenation/dictionaries',
-      './resources/shared/hyphenation',
-      '0d3b5e5314e68c3cf5d573b2e7bdc11143dcb821'
-    ).import
+    TwitterCldr::Resources::HyphenationImporter.new.import
   end
 
   desc 'Update README'

data/lib/twitter_cldr.rb

@@ -36,6 +36,9 @@ module TwitterCldr
   DEFAULT_CALENDAR_TYPE = :gregorian
 
   RESOURCES_DIR = File.join(File.dirname(File.dirname(File.expand_path(__FILE__))), 'resources')
+  VENDOR_DIR = File.join(File.dirname(File.dirname(File.expand_path(__FILE__))), 'vendor')
+  LIB_DIR = File.dirname(File.expand_path(__FILE__))
+  SPEC_DIR = File.join(File.dirname(File.dirname(File.expand_path(__FILE__))), 'spec')
 
   # TODO: convert this and all other hashes to 1.9 syntax
   # maps twitter locales to cldr locales

data/lib/twitter_cldr/formatters/numbers/rbnf.rb

@@ -29,6 +29,10 @@ module TwitterCldr
           rule_set: "spellout-numbering"
         }
 
+        def self.supported_locale?(locale)
+          TwitterCldr.resource_exists?('locales', locale, 'rbnf')
+        end
+
         attr_reader :locale
 
         def initialize(locale = TwitterCldr.locale)
@@ -118,7 +122,7 @@ module TwitterCldr
         end
 
         def resource
-          @resource ||= TwitterCldr.resources.get_locale_resource(locale, "rbnf")[locale][:rbnf][:grouping]
+          @resource ||= TwitterCldr.get_locale_resource(locale, "rbnf")[locale][:rbnf][:grouping]
         end
 
       end

data/lib/twitter_cldr/resources.rb

@@ -7,11 +7,10 @@ module TwitterCldr
   module Resources
     autoload :BidiTestImporter,               'twitter_cldr/resources/bidi_test_importer'
     autoload :CasefolderClassGenerator,       'twitter_cldr/resources/casefolder_class_generator'
-    autoload :CollationTriesDumper,           'twitter_cldr/resources/collation_tries_dumper'
-    autoload :CurrenciesImporter,             'twitter_cldr/resources/currencies_importer'
-    autoload :CustomLocalesResourcesImporter, 'twitter_cldr/resources/custom_locales_resources_importer'
+    autoload :CollationTriesImporter,         'twitter_cldr/resources/collation_tries_importer'
     autoload :HyphenationImporter,            'twitter_cldr/resources/hyphenation_importer'
-    autoload :IcuBasedImporter,               'twitter_cldr/resources/icu_based_importer'
+    autoload :Importer,                       'twitter_cldr/resources/importer'
+    autoload :ImportResolver,                 'twitter_cldr/resources/import_resolver'
     autoload :LanguageCodesImporter,          'twitter_cldr/resources/language_codes_importer'
     autoload :Loader,                         'twitter_cldr/resources/loader'
     autoload :LocalesResourcesImporter,       'twitter_cldr/resources/locales_resources_importer'
@@ -21,12 +20,94 @@ module TwitterCldr
     autoload :RbnfTestImporter,               'twitter_cldr/resources/rbnf_test_importer'
     autoload :ReadmeRenderer,                 'twitter_cldr/resources/readme_renderer'
     autoload :RegexpAstGenerator,             'twitter_cldr/resources/regexp_ast_generator'
+    autoload :Requirements,                   'twitter_cldr/resources/requirements'
     autoload :SegmentTestsImporter,           'twitter_cldr/resources/segment_tests_importer'
     autoload :TailoringImporter,              'twitter_cldr/resources/tailoring_importer'
     autoload :TransformTestImporter,          'twitter_cldr/resources/transform_test_importer'
     autoload :UnicodeDataImporter,            'twitter_cldr/resources/unicode_data_importer'
-    autoload :UnicodeImporter,                'twitter_cldr/resources/unicode_importer'
+    autoload :UnicodeFileParser,              'twitter_cldr/resources/unicode_file_parser'
     autoload :UnicodePropertyAliasesImporter, 'twitter_cldr/resources/unicode_property_aliases_importer'
     autoload :Uli,                            'twitter_cldr/resources/uli'
+
+    class << self
+      # these importer class methods aren't constants in order to avoid loading
+      # all the classes when the library is required
+
+      def standard_importer_classes
+        @standard_importer_classes ||= [
+          BidiTestImporter,
+          CasefolderClassGenerator,
+          CollationTriesImporter,
+          HyphenationImporter,
+          LanguageCodesImporter,
+          LocalesResourcesImporter,
+          PhoneCodesImporter,
+          PostalCodesImporter,
+          RbnfTestImporter,
+          SegmentTestsImporter,
+          TailoringImporter,
+          TransformTestImporter,
+          UnicodeDataImporter,
+          UnicodePropertyAliasesImporter,
+        ]
+      end
+
+      def uli_importer_classes
+        @uli_importer_classes ||= [
+          # Disabled for now since ULI TRAC has been down for quite a while.
+          # Word is data will eventually be available in a git repo.
+          # Uli::SegmentExceptionsImporter
+        ]
+      end
+
+      def property_importer_classes
+        @property_importer_classes ||= [
+          Properties::AgePropertyImporter,
+          Properties::ArabicShapingPropertyImporter,
+          Properties::BidiBracketsPropertyImporter,
+          Properties::BlocksPropertyImporter,
+          Properties::DerivedCorePropertiesImporter,
+          Properties::EastAsianWidthPropertyImporter,
+          Properties::GraphemeBreakPropertyImporter,
+          Properties::HangulSyllableTypePropertyImporter,
+          Properties::IndicPositionalCategoryPropertyImporter,
+          Properties::IndicSyllabicCategoryPropertyImporter,
+          Properties::JamoPropertyImporter,
+          Properties::LineBreakPropertyImporter,
+          Properties::PropListImporter,
+          Properties::ScriptExtensionsPropertyImporter,
+          Properties::ScriptPropertyImporter,
+          Properties::SentenceBreakPropertyImporter,
+          Properties::UnicodeDataPropertiesImporter,
+          Properties::WordBreakPropertyImporter
+        ]
+      end
+
+      def importer_classes
+        @importer_classes ||=
+          standard_importer_classes +
+          uli_importer_classes +
+          property_importer_classes
+      end
+
+      def importer_classes_for_ruby_engine
+        engine = case RUBY_ENGINE
+          when 'ruby' then :mri
+          when 'jruby' then :jruby
+          else
+            raise "Unsupported RUBY_ENGINE '#{RUBY_ENGINE}'"
+        end
+
+        importer_classes.select do |klass|
+          klass.default_params[:ruby_engine] == engine
+        end
+      end
+
+      def locale_based_importer_classes_for_ruby_engine
+        importer_classes_for_ruby_engine.select do |klass|
+          !!klass.default_params[:locales]
+        end
+      end
+    end
   end
 end

data/lib/twitter_cldr/resources/bidi_test_importer.rb

@@ -3,44 +3,50 @@
 # Copyright 2012 Twitter, Inc
 # http://www.apache.org/licenses/LICENSE-2.0
 
-require 'java'
-java_import "java.lang.Character"
-java_import "classpath.Bidi"
-
-require 'twitter_cldr/resources/download'
-
 module TwitterCldr
   module Resources
     # This class should be used with JRuby in 1.9 mode
 
-    class BidiTestImporter
-
-      BIDI_TEST_URL = 'http://unicode.org/Public/6.1.0/ucd/BidiTest.txt'  # this file is about 3.4 MB
-      OUT_FILE = "classpath_bidi_test.txt"
+    class BidiTestImporter < Importer
+      BIDI_TEST_FILE = 'ucd/BidiTest.txt'  # this file is about 3.4 MB
+      OUT_FILE = 'classpath_bidi_test.txt'
       DIRECTIONS = [nil, :LTR, :RTL]
 
-      def initialize(working_dir)
-        @working_dir = working_dir
-      end
+      requirement :unicode, '6.1.0', ['ucd/BidiTest.txt']
+      output_path File.join(TwitterCldr::SPEC_DIR, 'bidi')
+      ruby_engine :jruby
 
-      def import
-        TwitterCldr::Resources.download_if_necessary(File.join(@working_dir, File.basename(BIDI_TEST_URL)), BIDI_TEST_URL)
+      def execute
         generate_test
       end
 
-      protected
+      private
+
+      def before_prepare
+        require 'java'
+        java_import 'java.lang.Character'
+        java_import 'classpath.Bidi'
+      end
+
+      def source_file
+        requirements[:unicode].source_path_for(BIDI_TEST_FILE)
+      end
+
+      def output_file
+        File.join(params.fetch(:output_path), File.basename(BIDI_TEST_FILE))
+      end
 
       def generate_test
         run_hash = {}
 
-        File.open(File.join(@working_dir, File.basename(BIDI_TEST_URL)), "r").each_line do |ln|
+        File.open(source_file, 'r').each_line do |ln|
           cur_line = ln.strip
 
           case cur_line[0]
-            when "#", "@"
+            when '#', '@'
               next
             else
-              input, bitset = cur_line.split("; ")
+              input, bitset = cur_line.split('; ')
 
               expand_bitset_str(bitset).each_with_index do |check, index|
                 if check
@@ -48,7 +54,7 @@ module TwitterCldr
                   direction = get_java_direction(DIRECTIONS[index])
                   bidi = Java::Classpath::Bidi.new(types_to_string(types), direction)
                   levels = types.each_with_index.map { |_, idx| bidi.getLevelAt(idx) }
-                  reorder_arr = Java::Classpath::Bidi.reorderVisually(levels.dup, 0, (0...types.size).to_a, 0, types.size)
+                  reorder_arr = Java::Classpath::Bidi.reorderVisually(levels.dup, 0, (0...types.size).to_a, 0, types.size).to_a
 
                   key = "#{levels.join(" ")} | #{reorder_arr.join(" ")}"
                   run_hash[key] ||= {}
@@ -59,9 +65,9 @@ module TwitterCldr
           end
         end
 
-        File.open(File.join(@working_dir, OUT_FILE), "w+") do |out|
+        File.open(output_file, 'w+') do |out|
           run_hash.each_pair do |levels_and_reorders, inputs|
-            levels, reorders = levels_and_reorders.split(" | ")
+            levels, reorders = levels_and_reorders.split(' | ')
             out.write("@Levels: #{levels}\n")
             out.write("@Reorder: #{reorders}\n")
             inputs.each_pair do |input, bitset|
@@ -72,7 +78,7 @@ module TwitterCldr
       end
 
       def expand_bitset_str(bitset)
-        bitset.to_i.to_s(2).rjust(3, "0").chars.to_a.map { |i| i == "1" }.reverse
+        bitset.to_i.to_s(2).rjust(3, '0').chars.to_a.map { |i| i == '1' }.reverse
       end
 
       def get_java_direction(dir)
@@ -88,31 +94,31 @@ module TwitterCldr
 
       def types_to_string(types)
         @utf_map ||= {
-          "L"   => "\u0041",
-          "LRE" => "\u202a",
-          "LRO" => "\u202d",
-          "R"   => "\u05be",
-          "AL"  => "\u0626",
-          "RLE" => "\u202b",
-          "RLO" => "\u202e",
-          "PDF" => "\u202c",
-          "EN"  => "\u0030",
-          "ET"  => "\u0023",
-          "AN"  => "\u0667",
-          "CS"  => "\u002c",
-          "NSM" => "\u0300",
-          "BN"  => "\u0000",
-          "B"   => "\u0085",
-          "S"   => "\u0009",
-          "WS"  => "\u000c",
-          "ON"  => "\u0021"
+          'L'   => "\u0041",
+          'LRE' => "\u202a",
+          'LRO' => "\u202d",
+          'R'   => "\u05be",
+          'AL'  => "\u0626",
+          'RLE' => "\u202b",
+          'RLO' => "\u202e",
+          'PDF' => "\u202c",
+          'EN'  => "\u0030",
+          'ET'  => "\u0023",
+          'AN'  => "\u0667",
+          'CS'  => "\u002c",
+          'NSM' => "\u0300",
+          'BN'  => "\u0000",
+          'B'   => "\u0085",
+          'S'   => "\u0009",
+          'WS'  => "\u000c",
+          'ON'  => "\u0021"
         }
 
         # java 1.6 and 1.7 report different representative characters for the "ES" bidi class
-        @utf_map["ES"] = (Character.getDirectionality(0x002b) == Character::DIRECTIONALITY_EUROPEAN_NUMBER_SEPARATOR) ? "\u002b" : "\u002f"
-        types.inject("") { |ret, type| ret << @utf_map[type]; ret }
+        @utf_map['ES'] = (Character.getDirectionality(0x002b) == Character::DIRECTIONALITY_EUROPEAN_NUMBER_SEPARATOR) ? "\u002b" : "\u002f"
+        types.inject('') { |ret, type| ret << @utf_map[type]; ret }
       end
 
     end
   end
-end
+end