twitter_cldr 3.6.0 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cc6c6c92fe5dec780cafeb5fd8930add773c46d7
-  data.tar.gz: a7f4a52136cacf6e26d1b9bf0057dd19f9b852a8
+  metadata.gz: 77d562387667ce48ca9f7fd86dfbfe300c387342
+  data.tar.gz: 3269c97b2f7b47a84264c1c4766cd6fcf2205a97
 SHA512:
-  metadata.gz: 1192dfcd1168d55d7c30c586a8ece42f49d5d854e99a4ccb35e167c4cf7aa12c2b0c6040db5b08375acbc0532afbdad1888a62fa7a2c1ed57ae301706cbb8366
-  data.tar.gz: cc0b77e683c92a52af0ecb76a77192e93bb1157e1c428651822f3b713b844abe1ada7aa21b635663da67ac68606bdb88a15e7662594fff09a55c6b74c5c27736
+  metadata.gz: 2ae5ee0153fd3c1c12aa558f7315d815af5f592a4924674adc5c8786fc87d402c5e7db65f62d48a1fbf7df596dc23f0997126eaee622c1af42f9e1e214763097
+  data.tar.gz: f003b9f2514a86b1f1aa1af1ab1ffaf04c233f200b38e1d63a9bc52a80fc4f802ba2ea741602562d577a04f05086ec0234c8ed93f5b01cb570560a13452bfd2f

data/Gemfile

@@ -30,7 +30,8 @@ group :test do
 
   gem 'simplecov'
   gem 'launchy'
+end
 
+group :development, :test do
   gem 'benchmark-ips'
 end
-

data/History.txt

@@ -1,3 +1,10 @@
+== 4.0.0
+* Upgrade to Unicode v8.0.0, CLDR v29, and ICU 57.1.
+* Add support for fields.
+* Update plural rules with several bug fixes.
+* Add support for hyphenation (uses LibreOffice/Hunspell data).
+* Add support for transliteration.
+
 == 3.6.0
 * Override South Korean postal code format, which changed recently.
 

data/README.md

@@ -177,7 +177,7 @@ For English (and other languages), you can also specify an ordinal spellout:
 ```ruby
 DateTime.now.localize(:es).to_full_s               # "viernes, 14 de febrero de 2014, 12:20:05 (UTC +00:00)"
 DateTime.now.localize(:es).to_long_s               # "14 de febrero de 2014, 12:20:05 UTC"
-DateTime.now.localize(:es).to_medium_s             # "14 de feb. de 2014 12:20:05"
+DateTime.now.localize(:es).to_medium_s             # "14 feb. 2014 12:20:05"
 DateTime.now.localize(:es).to_short_s              # "14/2/14 12:20"
 
 Time.now.localize(:es).to_full_s                   # "12:20:05 (UTC +00:00)"
@@ -187,7 +187,7 @@ Time.now.localize(:es).to_short_s                  # "12:20"
 
 DateTime.now.localize(:es).to_date.to_full_s       # "viernes, 14 de febrero de 2014"
 DateTime.now.localize(:es).to_date.to_long_s       # "14 de febrero de 2014"
-DateTime.now.localize(:es).to_date.to_medium_s     # "14 de feb. de 2014"
+DateTime.now.localize(:es).to_date.to_medium_s     # "14 feb. 2014"
 DateTime.now.localize(:es).to_date.to_short_s      # "14/2/14"
 ```
 
@@ -237,22 +237,28 @@ It's important to know that, even though any given format may not be available a
 | H          | 12                     |
 | Hm         | 12:20                  |
 | Hms        | 12:20:05               |
+| Hmsv       | 12:20:05 v             |
+| Hmv        | 12:20 v                |
 | M          | 2                      |
 | MEd        | Fri, 2/14              |
 | MMM        | Feb                    |
 | MMMEd      | Fri, Feb 14            |
+| MMMMd      | February 14            |
 | MMMd       | Feb 14                 |
 | Md         | 2/14                   |
 | d          | 14                     |
 | h          | 12 PM                  |
 | hm         | 12:20 PM               |
 | hms        | 12:20:05 PM            |
+| hmsv       | 12:20:05 PM v          |
+| hmv        | 12:20 PM v             |
 | ms         | 20:05                  |
 | y          | 2014                   |
 | yM         | 2/2014                 |
 | yMEd       | Fri, 2/14/2014         |
 | yMMM       | Feb 2014               |
 | yMMMEd     | Fri, Feb 14, 2014      |
+| yMMMM      | February 2014          |
 | yMMMd      | Feb 14, 2014           |
 | yMd        | 2/14/2014              |
 | yQQQ       | Q1 2014                |
@@ -320,7 +326,7 @@ TwitterCldr::Localized::LocalizedTimespan.new(52).to_s(:approximate => true)  #
 CLDR contains a trove of calendar data, much of which can be accessed. One example is names of months, days, years.
 
 ```ruby
-TwitterCldr::Shared::Calendar.new(:sv).months.take(3) # ["Januari", "Februari", "Mars"]
+TwitterCldr::Shared::Calendar.new(:sv).months.take(3) # ["januari", "februari", "mars"]
 ```
 
 
@@ -490,14 +496,14 @@ TwitterCldr::Shared::Territories.all
 TwitterCldr::Shared::Territories.all_for(:pt)                                        # { ... :tl => "República Democrática de Timor-Leste", :tm => "Turcomenistão" ... }
 
 # get a territory by its code for the default locale
-TwitterCldr::Shared::Territories.from_territory_code(:'gb')                          # "U.K."
+TwitterCldr::Shared::Territories.from_territory_code(:'gb')                          # "UK"
 
 # get a territory from its code for a specific locale
 TwitterCldr::Shared::Territories.from_territory_code_for_locale(:gb, :pt)            # "Reino Unido"
 
 # translate a territory from one locale to another
 # signature: translate_territory(territory_name, source_locale, destination_locale)
-TwitterCldr::Shared::Territories.translate_territory("Reino Unido", :pt, :en)        # "U.K."
+TwitterCldr::Shared::Territories.translate_territory("Reino Unido", :pt, :en)        # "UK"
 TwitterCldr::Shared::Territories.translate_territory("U.K.", :en, :pt)               # "Reino Unido"
 ```
 
@@ -548,7 +554,7 @@ postal_code.regexp  # /(\d{5})(?:[ \-](\d{4}))?/
 Get a sample of valid postal codes with the `#sample` method:
 
 ```ruby
-postal_code.sample(5)  # ["40741-8626", "87304", "02763-2183", "78302", "37171"]
+postal_code.sample(5)  # ["49037", "77863", "11631-2555", "09849-9066", "60383"]
 ```
 
 ### Phone Codes
@@ -753,6 +759,37 @@ Convert code points to characters:
 TwitterCldr::Utils::CodePoints.to_string([0xBF])  # "¿"
 ```
 
+#### Unicode Properties
+
+Each character in the Unicode standard comes with a set of properties. Property data includes what type of script the character is written in, which version of the standard it was first introduced in, whether it represent a digit or an alphabetical symbol, its casing, and much more. Certain properties are boolean true/false values while others contain a set of property values. For example, the Hiragana letter "く" ("ku") has an `"Alphabetic"` property (i.e. Alphabetic = true) but does not have an `"Uppercase"` property (i.e. Uppercase = false). In addition, "く" has a property of `"Script"` with a property value for `"Script"` of `["Hiragana"]` (Note that property values are always arrays, since a single property can contain more than one property value).
+
+TwitterCLDR supports all the various Unicode properties and can look them up by character or retrieve a set of matching characters for the given property and property value. Let's use "く" again as an example. "く"'s Unicode code point is 12367 (i.e. `'く'.unpack("U*").first`):
+
+
+
+```ruby
+properties = TwitterCldr::Shared::CodePoint.get(12367).properties
+
+properties.alphabetic   # true
+properties.uppercase    # false
+properties.script.to_a  # ["Hiragana"]
+```
+
+Use `TwitterCldr::Shared::CodePoint.properties` to look up additional property information:
+
+
+
+
+
+
+```ruby
+properties = TwitterCldr::Shared::CodePoint.properties.code_points_for_property('Script', 'Hiragana')
+
+properties.to_a  # [12353..12438, 12445..12447 ... ]
+```
+
+Behind the scenes, these methods are using instances of `TwitterCldr::Shared::PropertiesDatabase`. Most of the time you probably won't need to use your own instance, but it is worth mentioning that `PropertiesDatabase` supplies methods for retrieving a list of available property names and normalizing property names and values. Finally, `TwitterCldr::Shared::CodePoint.properties` is an instance of `PropertiesDatabase` that you can use in place of creating a separate instance.
+
 #### Normalization
 
 Normalize/decompose a Unicode string (NFD, NFKD, NFC, and NFKC implementations available).  Note that the normalized string will almost always look the same as the original string because most character display systems automatically combine decomposed characters.
@@ -806,6 +843,42 @@ Turkic languages make use of the regular and dotted uppercase i characters "I" a
 "Istanbul".localize(:tr).casefold.to_s         # ıstanbul
 ```
 
+### Hyphenation
+
+TwitterCLDR uses data from the LibreOffice project to offer an implementation of [Franklin Liang's hyphenation algorithm](http://www.tug.org/docs/liang/). Try the `#hyphenate` method on instances of `LocalizedString`:
+
+```ruby
+'foolhardy undulate'.localize.hyphenate('-').to_s  # fool-hardy un-du-late
+```
+
+Since the data doesn't come packaged with CLDR, only a certain subset of locales are supported. To get a list of supported locales, use the `supported_locales` method:
+
+
+
+
+
+```ruby
+TwitterCldr::Shared::Hyphenator.supported_locales  # ["af-ZA", "de-CH", "en-US", ...]
+```
+
+You can also ask the `Hyphenator` class if a locale is supported:
+
+```ruby
+TwitterCldr::Shared::Hyphenator.supported_locale?(:en)  # true
+TwitterCldr::Shared::Hyphenator.supported_locale?(:ja)  # false
+```
+
+Create a new hyphenator instance via the `.get` method. If the locale is not supported, `.get` will raise an error.
+
+```ruby
+hyphenator = TwitterCldr::Shared::Hyphenator.get(:en)
+hyphenator.hyphenate('absolutely', '-')  # ab-so-lutely
+```
+
+The `.get` method will identify the best rule set to use by "maximizing" the given locale, a process that tries different combinations of the locale's language, region, and script based on CLDR's likely subtag data. In practice this means passing in `:en` works even though the hyphenator specifically supports en-US and en-GB.
+
+The second argument to `#hyphenate` is the delimiter to use, i.e. the hyphen text to insert at syllable boundaries. By default, the delimiter is the Unicode soft hyphen character (\u00AD). Depending on the system that's used to display the hyphenated text (word processor, browser, etc), the soft hyphen may render differently. Soft hyphens are supposed to be rendered only when the word needs to be displayed on multiple lines, and should be invisible otherwise.
+
 ### Sorting (Collation)
 
 TwitterCLDR contains an implementation of the [Unicode Collation Algorithm (UCA)](http://unicode.org/reports/tr10/) that provides language-sensitive text sorting capabilities.  Conveniently, all you have to do is use the `sort` method in combination with the familiar `localize` method.  Notice the difference between the default Ruby sort, which simply compares bytes, and the proper language-aware sort from TwitterCLDR in this German example:
@@ -840,6 +913,60 @@ collator.get_sort_key("Älg")             # [39, 61, 51, 1, 134, 157, 6, 1, 143,
 
 **Note**: The TwitterCLDR collator does not currently pass all the collation tests provided by Unicode, but for some strange reasons.  See the [summary](https://gist.github.com/f4ee3bd280a2257c5641) of these discrepancies if you're curious.
 
+### Transliteration
+
+Transliteration is the process of converting the text in one language or script into another with the goal of preserving the source language's pronunciation as much as possible. It can be useful in making text pronounceable in the target language. For example, most native English speakers would not be able to read or pronounce Japanese characters like these: "くろねこさま". Transliterating these characters into Latin script yields "kuronekosama", which should be pronounceable by most English speakers (in fact, probably speakers of many languages that use Latin characters). Remember, transliteration isn't translation; the actual meaning of the words is not taken into consideration, only the sound patterns.
+
+TwitterCLDR supports transliteration via the `#transliterate_into` method on `LocalizedString`. For example:
+
+```ruby
+"くろねこさま".localize.transliterate_into(:en)  # "kuronekosama"
+```
+
+This simple method hides quite a bit of complexity. First, TwitterCLDR identifies the scripts present in the source text. It then attempts to find any available transliterators to convert between the source language and the target language. If more than one transliterator is found, all of them will be applied to the source text.
+
+You can provide hints to the transliterator by providing additional locale information. For example, you can provide a source and target script:
+
+```ruby
+"くろねこさま".localize(:ja_Hiragana).transliterate_into(:en_Latin)  # "kuronekosama"
+```
+You may supply only the target script, only the source script, neither, or both. TwitterCLDR will try to find the best set of transliterators to get the job done.
+
+Behind the scenes, `LocalizedString#transliterate_into` creates instances of `TwitterCldr::Transforms::Transformer`. You can do this too if you're feeling adventurous. Here's our Japanese example again that uses `Transformer`:
+
+```ruby
+
+rule_set = TwitterCldr::Transforms::Transformer.get('Hiragana-Latin')
+rule_set.transform('くろねこさま')  # "kuronekosama"
+```
+Notice that the `.get` method was called with 'Hiragana-Latin' instead of 'ja-en' or something similar. This is because `.get` must be passed an exact transform id. To get a list of all supported transform ids, use the `Transformer#each_transform` method:
+
+
+
+
+
+```ruby
+TwitterCldr::Transforms::Transformer.each_transform.to_a  # ['Hiragana-Latin', 'Gujarati-Bengali', ...]
+```
+
+You can also search for transform ids using the `TransformId` class, which will attempt to find the closest matching transformer for the given source and target locales. Note that `.find` will return `nil` if no transformer can be found. You can pass instances of `TransformId` instead of a string when calling `Transformer.get`:
+
+
+
+
+
+
+```ruby
+TwitterCldr::Transforms::TransformId.find('ja', 'en')  # nil
+
+id = TwitterCldr::Transforms::TransformId.find('ja_Hiragana', 'en')
+id.source  # Hiragana
+id.target  # Latin
+
+rule_set = TwitterCldr::Transforms::Transformer.get(id)
+rule_set.transform('くろねこさま')  # "kuronekosama"
+```
+
 ### Handling Bidirectional Text
 
 When it comes to displaying text written in both right-to-left (RTL) and left-to-right (LTR) languages, most display systems run into problems.  The trouble is that Arabic or Hebrew text and English text (for example) often get scrambled visually and are therefore difficult to read.  It's not usually the basic ASCII characters like A-Z that get scrambled - it's most often punctuation marks and the like that are confusingly mixed up (they are considered "weak" types by Unicode).

data/Rakefile

@@ -17,7 +17,7 @@ require 'pry-nav'
 
 Bundler::GemHelper.install_tasks
 
-task :default => :spec
+task default: :spec
 
 desc 'Run specs'
 RSpec::Core::RakeTask.new do |t|
@@ -26,7 +26,7 @@ end
 
 namespace :spec do
   desc 'Run full specs suit'
-  task :full => [:full_spec_env, :spec]
+  task full: [:full_spec_env, :spec]
 
   task :full_spec_env do
     ENV['FULL_SPEC'] = 'true'
@@ -35,7 +35,7 @@ end
 
 namespace :spec do
   desc 'Run specs with SimpleCov'
-  task :cov => ['spec:simplecov_env', :spec] do
+  task cov: ['spec:simplecov_env', :spec] do
     require 'launchy'
     Launchy.open 'coverage/index.html'
   end
@@ -58,6 +58,7 @@ task :update do
       "update:tailoring_data",  # per locale
       "update:collation_tries", # per locale, must come after update:tailoring_data
       "update:rbnf_tests",      # per locale
+      "update:transform_tests",
       "update:bidi_tests"
     ]
   else
@@ -74,7 +75,8 @@ task :update do
       "update:language_codes",
       "update:segment_exceptions",     # per locale
       "update:segment_tests",
-      "update:readme"
+      "update:readme",
+      "update:hyphenation"
     ]
   end
 
@@ -196,6 +198,14 @@ namespace :update do
     ).import(TwitterCldr.supported_locales)
   end
 
+  desc 'Import (generate) transformation tests (should be executed using JRuby 1.7 in 1.9 mode)'
+  task :transform_tests, :icu4j_jar_path do |_, args|
+    TwitterCldr::Resources::TransformTestImporter.new(
+      './spec/transforms/test_data.yml',
+      args[:icu4j_jar_path] || ICU_JAR
+    ).import
+  end
+
   desc 'Import segment exceptions'
   task :segment_exceptions do
     TwitterCldr::Resources::Uli::SegmentExceptionsImporter.new(
@@ -212,6 +222,15 @@ namespace :update do
     ).import
   end
 
+  desc 'Import hyphenation dictionaries'
+  task :hyphenation_dictionaries do
+    TwitterCldr::Resources::HyphenationImporter.new(
+      './vendor/hyphenation/dictionaries',
+      './resources/shared/hyphenation',
+      '0d3b5e5314e68c3cf5d573b2e7bdc11143dcb821'
+    ).import
+  end
+
   desc 'Update README'
   task :readme do |_, args|
     renderer = TwitterCldr::Resources::ReadmeRenderer.new(

data/lib/twitter_cldr.rb

@@ -27,6 +27,7 @@ module TwitterCldr
   autoload :Shared,        'twitter_cldr/shared'
   autoload :Tokenizers,    'twitter_cldr/tokenizers'
   autoload :Utils,         'twitter_cldr/utils'
+  autoload :Transforms,    'twitter_cldr/transforms'
   autoload :Versions,      'twitter_cldr/versions'
 
   extend SingleForwardable
@@ -50,6 +51,8 @@ module TwitterCldr
 
   def_delegator :resources, :get_resource
   def_delegator :resources, :get_locale_resource
+  def_delegator :resources, :resource_exists?
+  def_delegator :resources, :absolute_resource_path
 
   class << self
 

data/lib/twitter_cldr/collation.rb

@@ -8,9 +8,8 @@ module TwitterCldr
     autoload :Collator,                  'twitter_cldr/collation/collator'
     autoload :ImplicitCollationElements, 'twitter_cldr/collation/implicit_collation_elements'
     autoload :SortKeyBuilder,            'twitter_cldr/collation/sort_key_builder'
-    autoload :Trie,                      'twitter_cldr/collation/trie'
     autoload :TrieBuilder,               'twitter_cldr/collation/trie_builder'
     autoload :TrieLoader,                'twitter_cldr/collation/trie_loader'
     autoload :TrieWithFallback,          'twitter_cldr/collation/trie_with_fallback'
   end
-end
+end

data/lib/twitter_cldr/collation/collator.rb

@@ -6,6 +6,8 @@
 module TwitterCldr
   module Collation
 
+    class UnexpectedCodePointError < StandardError; end
+
     # Collator uses fractional collation elements table form CLDR to generate sort keys for Unicode strings as well as
     # compare and sort such strings by generated sort keys.
     #
@@ -108,7 +110,13 @@ module TwitterCldr
         while non_starter_pos < code_points.size && !suffixes.empty?
           # get next code point (possibly non-starter)
           non_starter_code_point = code_points[non_starter_pos]
-          combining_class        = TwitterCldr::Shared::CodePoint.get(non_starter_code_point).combining_class.to_i
+          non_starter_metadata   = TwitterCldr::Shared::CodePoint.get(non_starter_code_point)
+
+          unless non_starter_metadata
+            raise UnexpectedCodePointError, "'#{non_starter_code_point}' does not appear to be a valid Unicode code point"
+          end
+
+          combining_class = non_starter_metadata.combining_class.to_i
 
           # code point is a starter or combining class has been already used (non-starter is 'blocked' from the prefix)
           break if combining_class == 0 || used_combining_classes[combining_class]

data/lib/twitter_cldr/collation/trie_builder.rb

@@ -30,7 +30,7 @@ module TwitterCldr
 
         private
 
-        def parse_collation_elements_table(table, trie = TwitterCldr::Collation::Trie.new)
+        def parse_collation_elements_table(table, trie = TwitterCldr::Utils::Trie.new)
           table.each_line do |line|
             trie.set(parse_code_points($1), parse_collation_element($2)) if COLLATION_ELEMENT_REGEXP =~ line
           end
@@ -80,4 +80,4 @@ module TwitterCldr
     end
 
   end
-end
+end

data/lib/twitter_cldr/collation/trie_with_fallback.rb

@@ -28,7 +28,7 @@ module TwitterCldr
     # by adding their one element prefixes to the trie itself. This feature is useful for some applications, e.g., for
     # suppressing contractions in a tailored fractional collation elements trie.
     #
-    class TrieWithFallback < TwitterCldr::Collation::Trie
+    class TrieWithFallback < TwitterCldr::Utils::Trie
 
       attr_accessor :fallback
 
@@ -54,4 +54,4 @@ module TwitterCldr
     end
 
   end
-end
+end

data/lib/twitter_cldr/localized.rb

@@ -6,7 +6,7 @@
 module TwitterCldr
   module Localized
     autoload :LocalizedArray,    'twitter_cldr/localized/localized_array'
-    autoload :LocalizedHash,    'twitter_cldr/localized/localized_hash'
+    autoload :LocalizedHash,     'twitter_cldr/localized/localized_hash'
     autoload :LocalizedDate,     'twitter_cldr/localized/localized_date'
     autoload :LocalizedDateTime, 'twitter_cldr/localized/localized_datetime'
     autoload :LocalizedNumber,   'twitter_cldr/localized/localized_number'
@@ -16,4 +16,4 @@ module TwitterCldr
     autoload :LocalizedTime,     'twitter_cldr/localized/localized_time'
     autoload :LocalizedTimespan, 'twitter_cldr/localized/localized_timespan'
   end
-end
+end

data/lib/twitter_cldr/localized/localized_date.rb

@@ -16,20 +16,8 @@ module TwitterCldr
         case @base_obj
           when Time
             LocalizedTime.new(@base_obj, @locale, chain_params)
-          when DateTime
-            utc_dt = @base_obj.new_offset(0)
-
-            time = Time.gm(
-              utc_dt.year,
-              utc_dt.month,
-              utc_dt.day,
-              utc_dt.hour,
-              utc_dt.min,
-              utc_dt.sec,
-              utc_dt.sec_fraction * 1_000_000
-            )
-
-            LocalizedTime.new(time, @locale, chain_params)
+          when Date, DateTime
+            LocalizedTime.new(@base_obj.to_time, @locale, chain_params)
           else
             nil
         end
@@ -57,4 +45,4 @@ module TwitterCldr
 
     end
   end
-end
+end