ffi-icu 0.6.1 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2f219be994f4a69799cf8b848c0909786476face806e223c89f8f4b455ef9b85
-  data.tar.gz: 562ceb9c82764d4251566107cadaabe7d0f63c69f012a0771c6ebf2d75677252
+  metadata.gz: 0f0256350b6cfd290356bcab5c7e501dcc55959f2a995dac452bb43756050788
+  data.tar.gz: 4fe1bd3bf83133f012f6d71d07ec997ceb9abae835e8b7be4bed6d8a7ea27bb1
 SHA512:
-  metadata.gz: ca6afe1397a8879f89303ed2b9866caaa906d0b4271b4fb75f8eb8a51e8e9a482f8b398072f13a85e5d790e9b261a7ab3178dda2c2a35a5dceb575fd798b5679
-  data.tar.gz: 9f722bd3b1248aeac053d74230ce9c98a9cde2d1e925a86284179880d969ec7d1fe59dbabf579306e9f63101bb013198ba6c637e4aa7e561a48b7995451e4411
+  metadata.gz: 8560b297f442ceb1548dc9214e7875cc050f2ca60bf9b9d7c8cc35ea1c2afcebbab52f139a294e610d6272049c2ca8de58fff1a9c8b42efded0088f7e6bf7a97
+  data.tar.gz: 18fd3d1b2c0f83ba1bc246381e41e72a77d8db495db2d1a0690b82c4d2cff7f00dbae88ab66f786d2d83c9445a925c1ddea2e6f7e6b235410344ac82a9a54fec

data/Gemfile

@@ -3,11 +3,10 @@ source 'https://rubygems.org'
 gemspec
 
 group :development, :test do
+  gem 'minitest', '~> 6.0'
+  gem 'minitest-mock', '~> 5.27'
   gem 'rake', '>= 12.3.3'
-  gem 'rspec', '~> 3.13'
   gem 'rubocop', '~> 1.86'
   gem 'rubocop-minitest', '~> 0.39'
   gem 'rubocop-packaging', '~> 0.6'
-  gem 'rubocop-rspec', '~> 3.9'
-  gem 'rubocop-rspec_rails', '~> 2.32'
 end

data/README.md

@@ -1,203 +1,404 @@
 # ffi-icu
 
-FFI wrappers for [International Components for Unicode (ICU)][icu].
-ICU provides comprehensive localization and security features.
-Majority personal computing devices, server operating systems and web browsers use ICU.
-ICU builds on top of Unicode's Common Locale Data Repository (CLDR).
+Ruby FFI wrappers for [International Components for Unicode (ICU)][icu].
+ICU provides Unicode text processing, character encoding detection, and
+locale-aware formatting. Its locale data comes from the
+[Common Locale Data Repository (CLDR)][cldr].
 
-## Gem
+## Installation
 
-[Rubygem](http://rubygems.org/gems/ffi-icu "ffi-icu")
+ffi-icu requires Ruby 3.2 or newer and the ICU shared libraries.
+On Linux, use the ICU packages supplied by a major distribution, such as Debian,
+Ubuntu, or Fedora. Install ICU from your distribution's supported repositories
+before installing the gem.
 
+### Debian and Ubuntu
+
+```sh
+sudo apt update
+sudo apt install libicu-dev
+```
+
+The `libicu-dev` package installs the matching ICU runtime libraries as dependencies.
+See the official [Debian package][debian-icu] and [Ubuntu package][ubuntu-icu] pages.
+
+### Fedora
+
+```sh
+sudo dnf install libicu
 ```
+
+See the official [Fedora libicu package][fedora-icu] page.
+
+### Install the gem
+
+```sh
 gem install ffi-icu
 ```
 
-## Dependencies
+The gem is published on [RubyGems][rubygems]. Load it before running the examples:
+
+```ruby
+require('ffi-icu')
+```
 
-ICU.
+### ICU in a custom directory
 
-If you get messages that the library or functions are not found, you can set some environment variables to tell ffi-icu where to find it, e.g.:
+ffi-icu normally discovers the distribution's ICU libraries automatically.
+If you need to override discovery, set `FFI_ICU_LIB` to the directory containing
+the shared libraries before loading the gem:
 
 ```sh
-$ export FFI_ICU_LIB="icui18n.so"
-$ export FFI_ICU_VERSION_SUFFIX="_3_8"
-$ ruby -r ffi-icu program.rb
+export FFI_ICU_LIB=/opt/icu/lib
+ruby my_script.rb
 ```
 
-# Features
+The library detects ICU's versioned function names automatically.
+
+## Features
 
-## Character Encoding Detection
+Localized output, including punctuation and spaces, can vary with the installed
+ICU and CLDR versions. Duration and list formatting require ICU 67 or newer.
 
-Examples:
+### Character Encoding Detection
+
+Detect an encoding and inspect the confidence score:
 
 ```ruby
-match = ICU::CharDet.detect(str)
+match = ICU::CharDet.detect('æåø')
 match.name       # => "UTF-8"
 match.confidence # => 80
 ```
 
-or
+Reuse a detector for multiple inputs:
 
 ```ruby
 detector = ICU::CharDet::Detector.new
-detector.detect(str) => #<struct ICU::CharDet::Detector::Match ...>
+detector.detect('æåø').name # => "UTF-8"
+detector.detect_all('foo bar').is_a?(Array) # => true
 ```
 
-* speed
+### Locale-Aware Collation
+
+Sort strings using a locale's collation rules:
 
-## Locale Sensitive Collation
+```ruby
+ICU::Collation.collate('nb', ['å', 'æ', 'ø']) # => ["æ", "ø", "å"]
+```
 
-Examples:
+Reuse a collator to compare and sort strings:
 
 ```ruby
-ICU::Collation.collate("nb", %w[å æ ø]) == %w[æ ø å] #=> true
+collator = ICU::Collation::Collator.new('nb')
+collator.compare('a', 'b')  # => -1
+collator.greater?('z', 'a') # => true
+collator.collate(['å', 'æ', 'ø']) # => ["æ", "ø", "å"]
 ```
 
-or
+### Text Boundary Analysis
+
+Find word boundaries or extract the substrings between them:
 
 ```ruby
-collator = ICU::Collation::Collator.new("nb")
-collator.compare("a", "b")  #=> -1
-collator.greater?("z", "a") #=> true
-collator.collate(%w[å æ ø]) #=> ["æ", "ø", "å"]
+iterator = ICU::BreakIterator.new(:word, 'en_US')
+iterator.text = 'This is a sentence.'
+iterator.to_a # => [0, 4, 5, 7, 8, 9, 10, 18, 19]
+iterator.substrings # => ["This", " ", "is", " ", "a", " ", "sentence", "."]
 ```
 
-## Text Boundary Analysis
+Boundary offsets use UTF-16 code units. Use `substrings` to extract text without
+converting those offsets to Ruby string indexes.
+
+### Number and Currency Formatting
 
-Examples:
+Format numbers, percentages, and currency amounts:
 
 ```ruby
-iterator = ICU::BreakIterator.new(:word, "en_US")
-iterator.text = "This is a sentence."
-iterator.to_a  #=> [0, 4, 5, 7, 8, 9, 10, 18, 19]
+ICU::NumberFormatting.format_number('en', 1_000) # => "1,000"
+ICU::NumberFormatting.format_number('de-DE', 1_234.56) # => "1.234,56"
+ICU::NumberFormatting.format_currency('en', 123.45, 'USD') # => "$123.45"
+ICU::NumberFormatting.format_percent('en', 0.53) # => "53%"
+ICU::NumberFormatting.spell('en_US', 1_000) # => "one thousand"
+ICU::NumberFormatting.format_number('en', BigDecimal('10000.123')) # => "10,000.123"
 ```
 
-## Number/Currency Formatting
+Reuse a formatter for repeated calls. The French Canadian example uses a
+non-breaking space (`\u00A0`) as its thousands separator:
 
-Examples:
+```ruby
+number_formatter = ICU::NumberFormatting.create('fr-CA')
+number_formatter.format(1_000) # => "1\u00A0000"
+number_formatter.format(1_000.123) # => "1\u00A0000,123"
+
+currency_formatter = ICU::NumberFormatting.create('en-US', :currency)
+currency_formatter.format(1_234.56, 'USD') # => "$1,234.56"
+```
+
+For anything the presets above do not cover, pass an
+[ICU number skeleton](https://unicode-org.github.io/icu/userguide/format_parse/numbers/skeletons.html)
+(ICU >= 62):
 
 ```ruby
-# class method interface
-ICU::NumberFormatting.format_number("en", 1_000) #=> "1,000"
-ICU::NumberFormatting.format_number("de-DE", 1234.56) #=> "1.234,56"
-ICU::NumberFormatting.format_currency("en", 123.45, 'USD') #=> "$123.45"
-ICU::NumberFormatting.format_percent("en", 0.53, 'USD') #=> "53%"
-ICU::NumberFormatting.spell("en_US", 1_000) #=> "one thousand"
+ICU::NumberFormatting.format_with_skeleton('en', 1_234, 'compact-short') # => "1.2K"
+ICU::NumberFormatting.format_with_skeleton('en', 0.1234, 'percent scale/100 .0') # => "12.3%"
+
+formatter = ICU::NumberFormatting.create_with_skeleton('en', 'precision-integer')
+formatter.format(999.6) # => "1,000"
+```
 
-# reusable formatting objects
-numf = ICU::NumberFormatting.create('fr-CA')
-numf.format(1000) #=> "1 000"
+### Currency Metadata
 
-curf = ICU::NumberFormatting.create('en-US', :currency)
-curf.format(1234.56, 'USD') #=> "$1,234.56"
+Look up ISO 4217 codes, localized names, symbols, and decimal digits:
+
+```ruby
+ICU::Currency.for_locale('en_US') # => "USD"
+ICU::Currency.for_locale('de_DE') # => "EUR"
+ICU::Currency.available(:common).include?('USD') # => true
+
+usd = ICU::Currency.new('usd')
+usd.code # => "USD"
+usd.symbol('en_US') # => "$"
+usd.name('en') # => "US Dollar"
+usd.plural_name('en', 'other') # => "US dollars"
+usd.fraction_digits # => 2
+usd.fraction_digits(:cash) # => 2
+usd.numeric_code # => 840
+
+ICU::Currency.new('JPY').fraction_digits # => 0
+ICU::Currency.symbol('EUR', 'en_US') # => "€"
 ```
 
-## Time Formatting/Parsing
+`narrow_symbol(locale)` requires ICU 61 or newer. `numeric_code` requires ICU 49 or newer.
+
+### Message Formatting
 
-Examples:
+Use numbered arguments for ICU MessageFormat patterns, including plurals and selects:
 
 ```ruby
-# class method interface
-f = ICU::TimeFormatting.format(Time.mktime(2015, 11, 12, 15, 21, 16), {:locale => 'cs_CZ', :zone => 'Europe/Prague', :date => :short, :time => :short})
-f #=> "12.11.15 15:21"
+formatter = ICU::MessageFormatting.create(
+  '{0, plural, one {# item} other {# items}}',
+  locale: 'en-US'
+)
+formatter.format([1]) # => "1 item"
+formatter.format([2]) # => "2 items"
+
+ICU::MessageFormatting.format('Hello, {0}!', locale: 'en-US', arguments: ['Ada'])
+# => "Hello, Ada!"
+```
+
+Validate a pattern before formatting:
 
-# reusable formatting objects
-formatter = ICU::TimeFormatting.create(:locale => 'cs_CZ', :zone => 'Europe/Prague', :date => :long, :time => :none)
-formatter.format(Time.now)  #=> "25. února 2015"
+```ruby
+pattern = '{0, select, male {He} female {She} other {They}}'
+ICU::MessageFormatting.validate(pattern, locale: 'en-US') # => true
+ICU::MessageFormatting.valid?('{0, plural', locale: 'en-US') # => false
 ```
 
+`validate` raises `ICU::MessageFormatting::InvalidPatternError` for an invalid
+pattern. The error exposes the pattern, locale, line, and offset. Use `valid?`
+when you want a boolean result.
+
+Arguments must be an array of numbered values. Supported types are `String`,
+`Integer`, `Float`, and `BigDecimal`. Use strings for simple and select arguments,
+and numbers for numeric arguments. Named arguments and rich-text tags are not supported.
+
+### Time Formatting and Parsing
+
+Set the time zone explicitly for predictable output:
+
 ```ruby
-# reusable formatting objects
-formatter = ICU::TimeFormatting.create(:locale => 'cs_CZ', :zone => 'Europe/Prague', :date => :long, :time => :none)
-formatter.parse("25. února 2015") #=> Wed Feb 25 00:00:00 +0100 2015
+time = Time.utc(2015, 11, 12, 14, 21)
+ICU::TimeFormatting.format(
+  time,
+  locale: 'cs_CZ', zone: 'Europe/Prague', date: :short, time: :short
+)
+# => "12.11.15 15:21"
 ```
 
-For skeleton formatting, visit the [Unicode date field symbol table](https://unicode-org.github.io/icu/userguide/format_parse/datetime/#date-field-symbol-table) page to help find the pattern characters to use.
+Reuse a formatter for formatting and parsing:
 
 ```ruby
-formatter = ICU::TimeFormatting.create(:locale => 'cs_CZ', :date => :pattern, :time => :pattern, :skeleton => 'MMMMY')
-formatter.format(Time.now)  #=> "únor 2015"
+formatter = ICU::TimeFormatting.create(
+  locale: 'cs_CZ', zone: 'Europe/Prague', date: :long, time: :none
+)
+formatter.format(Time.utc(2015, 2, 25)) # => "25. února 2015"
+formatter.parse('25. února 2015').utc.to_s # => "2015-02-24 23:00:00 UTC"
+```
+
+Use a skeleton to request date fields and let ICU choose the locale's pattern.
+See the [Unicode date field symbol table][date-fields] for supported symbols:
 
-formatter = ICU::TimeFormatting.create(:locale => 'cs_CZ', :date => :pattern, :time => :pattern, :skeleton => 'Y')
-formatter.format(Time.now)  #=> "2015"
+```ruby
+formatter = ICU::TimeFormatting.create(
+  locale: 'cs_CZ', zone: 'UTC', date: :pattern, time: :pattern, skeleton: 'MMMMy'
+)
+formatter.format(Time.utc(2015, 2, 25)) # => "únor 2015"
 ```
 
-## Duration Formatting
+Choose an hour cycle with `h11`, `h12`, `h23`, or `h24`, or use `:locale` for the
+locale's default:
 
 ```ruby
-# What the various styles look like
-formatter = ICU::DurationFormatting::DurationFormatter.new(locale: 'en-AU', style: :long)
-formatter.format({hours: 8, minutes: 40, seconds: 35})  #=> "8 hours, 40 minutes, 35 seconds"
+ICU::TimeFormatting.format(
+  Time.utc(2021, 4, 1, 12, 5),
+  locale: 'en_AU', zone: 'UTC', date: :none, time: :short, hour_cycle: 'h23'
+)
+# => "12:05"
+```
 
-formatter = ICU::DurationFormatting::DurationFormatter.new(locale: 'en-AU', style: :short)
-formatter.format({hours: 8, minutes: 40, seconds: 35})  #=> "8 hrs, 40 mins, 35 secs"
+### Duration Formatting
 
-formatter = ICU::DurationFormatting::DurationFormatter.new(locale: 'en-AU', style: :narrow)
-formatter.format({hours: 8, minutes: 40, seconds: 35})  #=> "8h 40min. 35s."
-formatter = ICU::DurationFormatting::DurationFormatter.new(locale: 'en-AU', style: :digital)
-formatter.format({hours: 8, minutes: 40, seconds: 35})  #=> "8:40:35"
+Duration formatting requires ICU 67 or newer. Choose `:long`, `:short`, `:narrow`,
+or `:digital`:
 
-# How digital & non-digital formats deal with units > hours
-formatter = ICU::DurationFormatting::DurationFormatter.new(locale: 'en-AU', style: :narrow)
-formatter.format({days: 2, hours: 8, minutes: 40, seconds: 35})  #=> "2d 8h 40min. 35s."
-formatter = ICU::DurationFormatting::DurationFormatter.new(locale: 'en-AU', style: :digital)
-formatter.format({days: 2, hours: 8, minutes: 40, seconds: 35})  #=> "2d 8:40:35"
+```ruby
+fields = { hours: 8, minutes: 40, seconds: 35 }
+ICU::DurationFormatting.format(fields, locale: 'en-AU', style: :long)
+# => "8 hours, 40 minutes, 35 seconds"
+ICU::DurationFormatting.format(fields, locale: 'en-AU', style: :short)
+# => "8 hrs, 40 mins, 35 secs"
+ICU::DurationFormatting.format(fields, locale: 'en-AU', style: :narrow)
+# => "8h 40m 35s"
+ICU::DurationFormatting.format(fields, locale: 'en-AU', style: :digital)
+# => "8:40:35"
+```
 
-# Missing or zero parts are omitted
-formatter = ICU::DurationFormatting::DurationFormatter.new(locale: 'en-AU', style: :long)
-formatter.format({days: 2, minutes: 40, seconds:0})  #=> "2 days, 40 minutes"
+Reuse a formatter when the locale and style stay the same. Larger units remain
+outside the digital clock, and missing or zero fields are omitted:
 
+```ruby
 formatter = ICU::DurationFormatting::DurationFormatter.new(locale: 'en-AU', style: :digital)
-formatter.format({hours: 2, minutes: 40})  #=> "2:40"
+formatter.format({ days: 2, hours: 8, minutes: 40, seconds: 35 }) # => "2d 8:40:35"
+formatter.format({ hours: 2, minutes: 40 }) # => "2:40"
+formatter.format({ minutes: 40, seconds: 7 }) # => "40:07"
 
-formatter = ICU::DurationFormatting::DurationFormatter.new(locale: 'en-AU', style: :digital)
-formatter.format({minutes: 40, seconds: 7})  #=> "40:07"
+formatter = ICU::DurationFormatting::DurationFormatter.new(locale: 'en-AU', style: :long)
+formatter.format({ days: 2, minutes: 40, seconds: 0 }) # => "2 days, 40 minutes"
+```
 
-# Sub-second parts are folded into seconds for digital display
-formatter = ICU::DurationFormatting::DurationFormatter.new(locale: 'en-AU', style: :digital)
-formatter.format({hours: 5, minutes: 7, seconds: 23, milliseconds: 98, microseconds: 997})  #=> "5:07:23.098997"
+Digital formatting folds milliseconds, microseconds, and nanoseconds into seconds.
+Supplying a smaller unit preserves its trailing zeros:
 
-# Zero-extension of sub-second parts in digital style
-formatter = ICU::DurationFormatting::DurationFormatter.new(locale: 'en-AU', style: :digital)
-formatter.format({hours: 5, minutes: 7, seconds: 23, milliseconds: 400})  #=> "5:07:23.400"
+```ruby
 formatter = ICU::DurationFormatting::DurationFormatter.new(locale: 'en-AU', style: :digital)
-formatter.format({hours: 5, minutes: 7, seconds: 23, milliseconds: 400, microseconds: 700})  #=> "5:07:23.400700"
+formatter.format({ hours: 5, minutes: 7, seconds: 23, milliseconds: 98, microseconds: 997 })
+# => "5:07:23.098997"
+formatter.format({ hours: 5, minutes: 7, seconds: 23, milliseconds: 400 })
+# => "5:07:23.400"
+formatter.format({ hours: 5, minutes: 7, seconds: 23, milliseconds: 400, microseconds: 700 })
+# => "5:07:23.400700"
+```
+
+Fractional parts are truncated except on the smallest supplied unit, which must
+be seconds or smaller to retain a fraction:
 
-# All fractional parts except the last are truncated
+```ruby
 formatter = ICU::DurationFormatting::DurationFormatter.new(locale: 'en-AU', style: :long)
-formatter.format({days: 2, hours: 7.3, minutes: 40.9, seconds:0.43})  #=> "2 days, 7 hours, 40 minutes, 0.43 seconds"
+formatter.format({ days: 2, hours: 7.3, minutes: 40.9, seconds: 0.43 })
+# => "2 days, 7 hours, 40 minutes, 0.43 seconds"
+```
 
-# With RU locale
-formatter = ICU::DurationFormatting::DurationFormatter.new(locale: 'ru', style: :long)
-formatter.format({hours: 1, minutes: 2, seconds: 3})  #=> "1 час 2 минуты 3 секунды"
+Unit names and separators follow the requested locale:
+
+```ruby
 formatter = ICU::DurationFormatting::DurationFormatter.new(locale: 'ru', style: :long)
-formatter.format({hours: 10, minutes: 20, seconds: 30})  #=> "10 часов 20 минут 30 секунд"
-formatter = ICU::DurationFormatting::DurationFormatter.new(locale: 'ru', style: :narrow)
-formatter.format({hours: 1, minutes: 2, seconds: 3})  #=> "1 ч 2 мин 3 с"
+formatter.format({ hours: 1, minutes: 2, seconds: 3 }) # => "1 час 2 минуты 3 секунды"
+formatter.format({ hours: 10, minutes: 20, seconds: 30 }) # => "10 часов 20 минут 30 секунд"
+
 formatter = ICU::DurationFormatting::DurationFormatter.new(locale: 'ru', style: :narrow)
-formatter.format({hours: 10, minutes: 20, seconds: 30})  #=> "10 ч 20 мин 30 с"
+formatter.format({ hours: 1, minutes: 2, seconds: 3 }) # => "1 ч 2 мин 3 с"
 ```
 
-## Transliteration
+### Transliteration
 
-Example:
+Convert between scripts or reuse a transliterator for multiple inputs:
 
 ```ruby
 ICU::Transliteration.transliterate('Traditional-Simplified', '沈從文') # => "沈从文"
+
+transliterator = ICU::Transliteration::Transliterator.new('Latin-ASCII')
+transliterator.transliterate('café') # => "cafe"
+```
+
+Apply a sequence of custom transliteration rules:
+
+```ruby
+ICU::Transliteration.translit('NFD; [:Nonspacing Mark:] Remove; NFC', 'âêîôû') # => "aeiou"
 ```
 
-## Locale
+### Unicode Normalization
 
-Examples:
+Compose canonically equivalent text into NFC, or check whether it is already normalized:
+
+```ruby
+normalizer = ICU::Normalizer.new(nil, 'nfc', :compose)
+normalizer.normalize("A\u030A") # => "Å"
+normalizer.normalized?('Å') # => true
+normalizer.normalized?("A\u030A") # => false
+```
+
+Use `:decompose` with `'nfc'` for NFD. Use `'nfkc'` with `:compose` or `:decompose`
+for compatibility normalization (NFKC or NFKD):
+
+```ruby
+normalizer = ICU::Normalizer.new(nil, 'nfkc', :compose)
+normalizer.normalize('ffi') # => "ffi"
+```
+
+A convenience method also supports one-off normalization:
+
+```ruby
+ICU::Normalization.normalize('Å', :nfd) # => "A\u030A"
+ICU::Normalization.normalize("A\u030A", :nfc) # => "Å"
+```
+
+### List Formatting
+
+List formatting requires ICU 67 or newer. Join strings with locale-aware
+separators and conjunctions:
+
+```ruby
+ICU::ListFormatting.format(['Alice', 'Bob', 'Jane'], locale: 'en') # => "Alice, Bob, and Jane"
+ICU::ListFormatting.format(['Alice', 'Bob', 'Jane'], locale: 'fr', style: :and) # => "Alice, Bob et Jane"
+ICU::ListFormatting.format(['Alice', 'Bob', 'Jane'], locale: 'en', style: :or) # => "Alice, Bob, or Jane"
+ICU::ListFormatting.format(['Alice', 'Bob', 'Jane'], locale: 'en', style: :unit) # => "Alice, Bob, Jane"
+```
+
+The default style is `:and`. Styles must be symbols, and all items must be strings.
+The formatter uses ICU's wide list width.
+
+### Locale
+
+Look up localized names and choose display-name contexts:
 
 ```ruby
 locale = ICU::Locale.new('en-US')
-locale.display_country('en-US') #=> "United States"
-locale.display_language('es') #=> "inglés"
-locale.display_name('es') #=> "inglés (Estados Unidos)"
-locale.display_name_with_context('en-US', [:length_short]) #=> "English (US)"
-locale.display_name_with_context('en-US', [:length_long])  #=> "English (United States)"
+locale.display_country('en-US') # => "United States"
+locale.display_language('es') # => "inglés"
+locale.display_name('es') # => "inglés (Estados Unidos)"
+locale.display_name_with_context('en-US', [:length_short]) # => "English (US)"
+locale.display_name_with_context('en-US', [:length_full]) # => "English (United States)"
+
+ICU::Locale.new('fa-AF').display_name_with_context('en', [:dialect_names]) # => "Dari"
 ```
 
+Convert BCP 47 language tags to ICU locale IDs or add likely subtags:
+
+```ruby
+locale = ICU::Locale.for_language_tag('en-US')
+locale.id # => "en_US"
+locale.to_language_tag # => "en-US"
+ICU::Locale.new('en').with_likely_subtags.to_s # => "en_Latn_US"
+```
+
+## Benchmarks
+
+Benchmarks live in [`benchmark/`](benchmark/README.md) with a separate Gemfile.
+
 [icu]: https://github.com/unicode-org/icu
+[cldr]: https://cldr.unicode.org/
+[rubygems]: https://rubygems.org/gems/ffi-icu
+[debian-icu]: https://packages.debian.org/stable/libicu-dev
+[ubuntu-icu]: https://packages.ubuntu.com/search?keywords=libicu-dev
+[fedora-icu]: https://packages.fedoraproject.org/pkgs/icu/libicu/
+[date-fields]: https://unicode-org.github.io/icu/userguide/format_parse/datetime/#date-field-symbol-table

data/Rakefile

@@ -4,18 +4,14 @@ require 'rake'
 require 'bundler'
 Bundler::GemHelper.install_tasks
 
-require 'rspec/core/rake_task'
-RSpec::Core::RakeTask.new(:spec) do |spec|
-  spec.pattern = 'spec/**/*_spec.rb'
-  spec.rspec_opts = '--backtrace'
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'test' << 'lib'
+  t.pattern = 'test/**/*_test.rb'
+  t.warning = false
 end
 
-RSpec::Core::RakeTask.new(:rcov) do |spec|
-  spec.pattern = 'spec/**/*_spec.rb'
-  spec.rcov = true
-end
-
-task default: :spec
+task default: :test
 
 begin
   require('yard')

data/lib/ffi-icu/break_iterator.rb

@@ -20,16 +20,20 @@ module ICU
     end
 
     def text=(str)
-      @text = str
+      text_pointer = UCharPointer.from_utf8(str)
 
       Lib.check_error do |err|
-        Lib.ubrk_setText(@iterator, UCharPointer.from_string(str), str.size, err)
+        Lib.ubrk_setText(@iterator, text_pointer, text_pointer.length_in_uchars, err)
       end
+
+      @text = str
+      @text_pointer = text_pointer
     end
 
     def each
       return to_enum(:each) unless block_given?
 
+      # Positional methods expose ICU UTF-16 code-unit offsets.
       int = first
 
       while int != DONE
@@ -43,11 +47,13 @@ module ICU
     def each_substring
       return to_enum(:each_substring) unless block_given?
 
-      # each_char needed for 1.8, where String#[] works on bytes, not characters
       chars = text.each_char.to_a
-      low   = first
+      utf16_to_ruby = ruby_offsets_for(chars)
+
+      low = utf16_to_ruby[first]
 
       while (high = self.next) != DONE
+        high = utf16_to_ruby[high]
         yield(chars[low...high].join)
         low = high
       end
@@ -55,6 +61,19 @@ module ICU
       self
     end
 
+    def ruby_offsets_for(chars)
+      offsets = [0]
+      utf16_offset = 0
+
+      chars.each_with_index do |char, ruby_index|
+        utf16_offset += char.encode(Encoding::UTF_16LE).bytesize / UCharPointer::TYPE_SIZE
+        offsets[utf16_offset] = ruby_index + 1
+      end
+
+      offsets
+    end
+    private :ruby_offsets_for
+
     def substrings
       each_substring.to_a
     end
@@ -64,7 +83,7 @@ module ICU
     end
 
     def previous
-      Lib.ubrk_next(@iterator)
+      Lib.ubrk_previous(@iterator)
     end
 
     def first