icu4x 0.5.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 07dd6b700780e96f64b0f5925d1a7340a44b70c77112f863f018494c8f55db77
+  data.tar.gz: 614f4b010e2ce82a1e477122e322b8ecf258708e61db00efb02d06bc1f5d74f1
+SHA512:
+  metadata.gz: 162056347baf80f0d262fba295d8dd1a2efad853b1dc267ee060c5537ff28a25c42efbee884f1694eac8536561da495eac1eb3b67b9419e3cd3a274b2a328e92
+  data.tar.gz: a43251139b2778e922d6977875179aea1f59b25ac1c7a8e01f5d98cafdc047714ea901ae9926250f2d96259f6a0fa29263d0dbc970ad7632abce27617de66fbb

data/CHANGELOG.md

@@ -0,0 +1,17 @@
+## [Unreleased]
+
+## [0.5.0] - 2026-01-01
+
+### Added
+
+- **Locale**: BCP 47 (`parse`) and POSIX (`parse_posix`) locale parsing
+- **DateTimeFormat**: Date/time formatting with timezone and calendar support
+- **NumberFormat**: Decimal, currency, and percent formatting with configurable options
+- **PluralRules**: Cardinal and ordinal plural category selection
+- **Collator**: Locale-sensitive string comparison with sensitivity and numeric options
+- **ListFormat**: Conjunction, disjunction, and unit list formatting
+- **RelativeTimeFormat**: Relative time formatting for all time units
+- **DisplayNames**: Localized names for languages, regions, scripts, and locales
+- **Segmenter**: Text segmentation by grapheme, word, sentence, or line
+- **DataProvider**: Blob data loading with automatic locale fallback
+- **DataGenerator**: CLDR data export with locale and marker filtering

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2025 OZAWA Sakuro
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,118 @@
+# ICU4X
+
+Ruby bindings for [ICU4X](https://github.com/unicode-org/icu4x), providing internationalization functionality.
+
+## Overview
+
+ICU4X is a Unicode library that provides locale-aware formatting and processing. This gem wraps ICU4X via Rust extensions, offering an API similar to JavaScript's Intl.
+
+No locale data is bundled with the gem. Users generate only the data they need, keeping applications lean.
+
+## Features
+
+- **Locale** - BCP 47 and POSIX locale identifier parsing
+- **DateTimeFormat** - Locale-aware date/time formatting with timezone and calendar support
+- **NumberFormat** - Number, currency, and percent formatting
+- **PluralRules** - CLDR plural category selection (cardinal/ordinal)
+- **Collator** - Locale-sensitive string comparison and sorting
+- **ListFormat** - Locale-aware list formatting (and, or, unit)
+- **RelativeTimeFormat** - Relative time formatting (e.g., "3 days ago")
+- **DisplayNames** - Localized names for languages, regions, and scripts
+- **Segmenter** - Text segmentation (grapheme, word, sentence, line)
+- **DataProvider** - Locale data loading with automatic fallback
+- **DataGenerator** - Locale data generation from CLDR
+
+## Requirements
+
+- Ruby 3.2+
+- Rust toolchain (for building the native extension)
+
+## Setup
+
+Add to your Gemfile:
+
+```ruby
+gem "icu4x"
+```
+
+Then generate locale data for your application:
+
+```ruby
+require "icu4x"
+
+ICU4X::DataGenerator.export(
+  locales: %w[en ja],
+  markers: :all,
+  format: :blob,
+  output: Pathname.new("data/i18n.blob")
+)
+```
+
+## Usage
+
+```ruby
+require "icu4x"
+
+# Option 1: Configure default provider (recommended)
+ICU4X.configure do |config|
+  config.data_path = Pathname.new("data/i18n.blob")
+end
+# Or set ICU4X_DATA_PATH environment variable
+
+# Option 2: Load provider explicitly
+provider = ICU4X::DataProvider.from_blob(Pathname.new("data/i18n.blob"))
+
+# Parse locale (BCP 47 or POSIX format)
+locale = ICU4X::Locale.parse("ja-JP")
+locale = ICU4X::Locale.parse_posix("ja_JP.UTF-8")  # POSIX format also supported
+
+# Date/time formatting (provider: can be omitted if default is configured)
+dtf = ICU4X::DateTimeFormat.new(locale, provider:, date_style: :long)
+dtf.format(Time.now)
+# => "2025年12月30日"
+
+# Number formatting
+nf = ICU4X::NumberFormat.new(locale, provider:, style: :currency, currency: "JPY")
+nf.format(1_234_567)
+# => "￥1,234,567"
+
+# Plural rules
+pr = ICU4X::PluralRules.new(ICU4X::Locale.parse("en"), provider:)
+pr.select(1)   # => :one
+pr.select(2)   # => :other
+
+# Collation (sorting)
+collator = ICU4X::Collator.new(locale, provider:)
+%w[メロン アップル なし].sort { |a, b| collator.compare(a, b) }
+# => ["アップル", "なし", "メロン"]
+
+# List formatting
+lf = ICU4X::ListFormat.new(locale, provider:, type: :conjunction)
+lf.format(%w[Apple Banana Cherry])
+# => "Apple、Banana、Cherry"
+
+# Relative time formatting
+rtf = ICU4X::RelativeTimeFormat.new(locale, provider:)
+rtf.format(-3, :day)
+# => "3日前"
+
+# Display names
+dn = ICU4X::DisplayNames.new(locale, provider:, type: :language)
+dn.of("en")
+# => "英語"
+
+# Text segmentation
+segmenter = ICU4X::Segmenter.new(granularity: :word, provider:)
+segmenter.segment("Hello, world!").map(&:text)
+# => ["Hello", ",", " ", "world", "!"]
+```
+
+See [doc/](doc/) for detailed documentation.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/sakuro/icu4x.
+
+## License
+
+MIT License. See [LICENSE](LICENSE.txt) for details.

data/ext/icu4x/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "icu4x"
+version = "0.1.0"
+edition = "2024"
+publish = false
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+magnus = "0.8"
+icu_locale = "2.1"
+icu_provider = "2.1"
+icu_provider_blob = { version = "2.1", features = ["alloc", "export"] }
+icu_provider_source = { version = "2.1", features = ["networking", "experimental"] }
+icu_provider_export = "2.1"
+icu_provider_registry = "2.1"
+icu_provider_adapters = "2.1"
+icu = { version = "2.1", features = ["experimental"] }
+fixed_decimal = "0.7"
+tinystr = "0.8"
+jiff = "0.2"
+icu4x_macros = { path = "../icu4x_macros" }

data/ext/icu4x/extconf.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+require "mkmf"
+require "rb_sys/mkmf"
+
+create_rust_makefile("icu4x/icu4x")

data/ext/icu4x/src/collator.rs

@@ -0,0 +1,207 @@
+use crate::data_provider::DataProvider;
+use crate::helpers;
+use icu::collator::Collator as IcuCollator;
+use icu::collator::CollatorPreferences;
+use icu::collator::options::{CaseLevel, CollatorOptions, Strength};
+use icu::collator::preferences::{CollationCaseFirst, CollationNumericOrdering};
+use icu_provider::buf::AsDeserializingBufferProvider;
+use icu4x_macros::RubySymbol;
+use magnus::{
+    Error, RHash, RModule, Ruby, TryConvert, Value, function, method, prelude::*,
+};
+use std::cmp::Ordering;
+
+/// Sensitivity level for collation
+#[derive(Clone, Copy, PartialEq, Eq, RubySymbol)]
+enum Sensitivity {
+    Base,
+    Accent,
+    Case,
+    Variant,
+}
+
+/// Case first option
+#[derive(Clone, Copy, PartialEq, Eq, RubySymbol)]
+enum CaseFirstOption {
+    Upper,
+    Lower,
+}
+
+impl CaseFirstOption {
+    fn to_icu_case_first(self) -> CollationCaseFirst {
+        match self {
+            CaseFirstOption::Upper => CollationCaseFirst::Upper,
+            CaseFirstOption::Lower => CollationCaseFirst::Lower,
+        }
+    }
+}
+
+/// Ruby wrapper for ICU4X Collator
+#[magnus::wrap(class = "ICU4X::Collator", free_immediately, size)]
+pub struct Collator {
+    inner: IcuCollator,
+    locale_str: String,
+    sensitivity: Sensitivity,
+    numeric: bool,
+    case_first: Option<CaseFirstOption>,
+}
+
+// SAFETY: Ruby's GVL protects access to this type.
+unsafe impl Send for Collator {}
+
+impl Collator {
+    /// Create a new Collator instance
+    ///
+    /// # Arguments
+    /// * `locale` - A Locale instance
+    /// * `provider:` - A DataProvider instance
+    /// * `sensitivity:` - :base, :accent, :case, or :variant (default)
+    /// * `numeric:` - Whether to use numeric sorting (default: false)
+    /// * `case_first:` - :upper, :lower, or nil (default)
+    fn new(ruby: &Ruby, args: &[Value]) -> Result<Self, Error> {
+        // Parse arguments: (locale, **kwargs)
+        let (icu_locale, locale_str) = helpers::extract_locale(ruby, args)?;
+
+        // Get kwargs (optional)
+        let kwargs: RHash = if args.len() > 1 {
+            TryConvert::try_convert(args[1])?
+        } else {
+            ruby.hash_new()
+        };
+
+        // Resolve provider: use explicit or fall back to default
+        let resolved_provider = helpers::resolve_provider(ruby, &kwargs)?;
+
+        // Extract sensitivity option (default: :variant)
+        let sensitivity =
+            helpers::extract_symbol(ruby, &kwargs, "sensitivity", Sensitivity::from_ruby_symbol)?
+                .unwrap_or(Sensitivity::Variant);
+
+        // Extract numeric option (default: false)
+        let numeric: bool = kwargs
+            .lookup::<_, Option<bool>>(ruby.to_symbol("numeric"))?
+            .unwrap_or(false);
+
+        // Extract case_first option (default: nil)
+        let case_first = helpers::extract_symbol(
+            ruby,
+            &kwargs,
+            "case_first",
+            CaseFirstOption::from_ruby_symbol,
+        )?;
+
+        // Get the error exception class
+        let error_class = helpers::get_exception_class(ruby, "ICU4X::Error");
+
+        // Get the DataProvider
+        let dp: &DataProvider = TryConvert::try_convert(resolved_provider).map_err(|_| {
+            Error::new(
+                ruby.exception_type_error(),
+                "provider must be a DataProvider",
+            )
+        })?;
+
+        // Build collator options (strength and case_level)
+        let mut options = CollatorOptions::default();
+
+        // Set strength based on sensitivity
+        options.strength = Some(match sensitivity {
+            Sensitivity::Base => Strength::Primary,
+            Sensitivity::Accent => Strength::Secondary,
+            Sensitivity::Case => Strength::Primary,
+            Sensitivity::Variant => Strength::Tertiary,
+        });
+
+        // Set case_level for case sensitivity
+        if matches!(sensitivity, Sensitivity::Case) {
+            options.case_level = Some(CaseLevel::On);
+        }
+
+        // Build preferences (numeric and case_first)
+        let mut prefs: CollatorPreferences = (&icu_locale).into();
+
+        if numeric {
+            prefs.numeric_ordering = Some(CollationNumericOrdering::True);
+        }
+
+        if let Some(cf) = case_first {
+            prefs.case_first = Some(cf.to_icu_case_first());
+        }
+
+        // Create collator
+        let collator = IcuCollator::try_new_unstable(&dp.inner.as_deserializing(), prefs, options)
+            .map_err(|e| Error::new(error_class, format!("Failed to create Collator: {}", e)))?;
+
+        Ok(Self {
+            inner: collator,
+            locale_str,
+            sensitivity,
+            numeric,
+            case_first,
+        })
+    }
+
+    /// Compare two strings
+    ///
+    /// # Arguments
+    /// * `a` - First string
+    /// * `b` - Second string
+    ///
+    /// # Returns
+    /// -1 if a < b, 0 if a == b, 1 if a > b
+    fn compare(&self, a: Value, b: Value) -> Result<i32, Error> {
+        let ruby = Ruby::get().expect("Ruby runtime should be available");
+
+        let str_a: String = TryConvert::try_convert(a).map_err(|_| {
+            Error::new(
+                ruby.exception_type_error(),
+                "first argument must be a String",
+            )
+        })?;
+
+        let str_b: String = TryConvert::try_convert(b).map_err(|_| {
+            Error::new(
+                ruby.exception_type_error(),
+                "second argument must be a String",
+            )
+        })?;
+
+        let result = match self.inner.as_borrowed().compare(&str_a, &str_b) {
+            Ordering::Less => -1,
+            Ordering::Equal => 0,
+            Ordering::Greater => 1,
+        };
+
+        Ok(result)
+    }
+
+    /// Get the resolved options
+    ///
+    /// # Returns
+    /// A hash with :locale, :sensitivity, :numeric, and optionally :case_first
+    fn resolved_options(&self) -> Result<RHash, Error> {
+        let ruby = Ruby::get().expect("Ruby runtime should be available");
+        let hash = ruby.hash_new();
+        hash.aset(ruby.to_symbol("locale"), self.locale_str.as_str())?;
+        hash.aset(
+            ruby.to_symbol("sensitivity"),
+            ruby.to_symbol(self.sensitivity.to_symbol_name()),
+        )?;
+        hash.aset(ruby.to_symbol("numeric"), self.numeric)?;
+        if let Some(cf) = self.case_first {
+            hash.aset(
+                ruby.to_symbol("case_first"),
+                ruby.to_symbol(cf.to_symbol_name()),
+            )?;
+        }
+        Ok(hash)
+    }
+}
+
+pub fn init(ruby: &Ruby, module: &RModule) -> Result<(), Error> {
+    let class = module.define_class("Collator", ruby.class_object())?;
+    class.define_singleton_method("new", function!(Collator::new, -1))?;
+    class.define_method("compare", method!(Collator::compare, 2))?;
+    class.define_method("resolved_options", method!(Collator::resolved_options, 0))?;
+    Ok(())
+}

data/ext/icu4x/src/data_generator.rs

@@ -0,0 +1,244 @@
+use crate::helpers;
+use icu_provider::DataMarkerInfo;
+use icu_provider_blob::export::BlobExporter;
+use icu_provider_export::prelude::*;
+use icu_provider_source::SourceDataProvider;
+use magnus::{
+    Error, RArray, RClass, RHash, RModule, Ruby, Symbol, Value, function, prelude::*,
+    value::ReprValue,
+};
+use std::collections::HashMap;
+use std::fs::File;
+use std::io::BufWriter;
+use std::path::PathBuf;
+use std::sync::OnceLock;
+
+/// Returns a static lookup table mapping marker names to DataMarkerInfo
+fn marker_lookup() -> &'static HashMap<&'static str, DataMarkerInfo> {
+    static LOOKUP: OnceLock<HashMap<&'static str, DataMarkerInfo>> = OnceLock::new();
+    LOOKUP.get_or_init(|| {
+        let mut map = HashMap::new();
+        macro_rules! cb {
+            ($($marker_ty:ty:$marker:ident,)+ #[experimental] $($emarker_ty:ty:$emarker:ident,)+) => {
+                $(
+                    // Add both the full type name and the short marker name
+                    map.insert(stringify!($marker_ty), <$marker_ty>::INFO);
+                    map.insert(stringify!($marker), <$marker_ty>::INFO);
+                )+
+                // Also include experimental markers
+                $(
+                    map.insert(stringify!($emarker_ty), <$emarker_ty>::INFO);
+                    map.insert(stringify!($emarker), <$emarker_ty>::INFO);
+                )+
+            };
+        }
+        icu_provider_registry::registry!(cb);
+        map
+    })
+}
+
+/// Ruby wrapper for ICU4X data generation functionality
+pub struct DataGenerator;
+
+impl DataGenerator {
+    /// Export ICU4X data to a blob file
+    ///
+    /// # Arguments
+    /// * `locales` - Array of locale strings (e.g., ["ja", "en"])
+    /// * `markers` - :all or Array of marker symbols (e.g., [:datetime, :number, :plurals])
+    /// * `format` - :blob (only blob format is supported)
+    /// * `output` - Pathname for the output file
+    fn export(ruby: &Ruby, kwargs: RHash) -> Result<(), Error> {
+        // Extract locales
+        let locales_value: RArray = kwargs
+            .fetch::<_, RArray>(ruby.to_symbol("locales"))
+            .map_err(|_| {
+                Error::new(
+                    ruby.exception_arg_error(),
+                    "missing required keyword argument: locales",
+                )
+            })?;
+
+        let mut locale_families: Vec<DataLocaleFamily> = Vec::new();
+        let mut has_und = false;
+        for i in 0..locales_value.len() {
+            let locale_str: String = locales_value.entry(i as isize)?;
+            if locale_str == "und" {
+                has_und = true;
+            }
+            let family = DataLocaleFamily::with_descendants(locale_str.parse().map_err(|e| {
+                Error::new(
+                    ruby.exception_arg_error(),
+                    format!("Invalid locale '{}': {}", locale_str, e),
+                )
+            })?);
+            locale_families.push(family);
+        }
+
+        // Warn if 'und' locale is not included
+        if !has_und {
+            let kernel: Value = ruby.eval("Kernel")?;
+            let _: Value = kernel.funcall(
+                "warn",
+                ("ICU4X::DataGenerator.export: 'und' locale not included. Fallback may fail for unlisted locales.",),
+            )?;
+        }
+
+        // Extract markers
+        let markers_value: Value = kwargs
+            .fetch::<_, Value>(ruby.to_symbol("markers"))
+            .map_err(|_| {
+                Error::new(
+                    ruby.exception_arg_error(),
+                    "missing required keyword argument: markers",
+                )
+            })?;
+
+        // Parse markers: either :all or an array of marker name strings
+        let selected_markers: Option<Vec<DataMarkerInfo>> = {
+            let all_symbol = ruby.to_symbol("all");
+            if markers_value.eql(all_symbol)? {
+                // :all - use all available markers (None means don't filter)
+                None
+            } else if let Ok(markers_array) = RArray::try_convert(markers_value) {
+                // Array of marker name strings
+                let lookup = marker_lookup();
+                let mut result = Vec::new();
+                for i in 0..markers_array.len() {
+                    let marker_name: String = markers_array.entry(i as isize)?;
+                    match lookup.get(marker_name.as_str()) {
+                        Some(&info) => result.push(info),
+                        None => {
+                            return Err(Error::new(
+                                ruby.exception_arg_error(),
+                                format!(
+                                    "unknown marker: '{}'. Use DataGenerator.available_markers to see valid names.",
+                                    marker_name
+                                ),
+                            ));
+                        }
+                    }
+                }
+                Some(result)
+            } else {
+                return Err(Error::new(
+                    ruby.exception_arg_error(),
+                    "markers must be :all or an Array of marker name strings",
+                ));
+            }
+        };
+
+        // Extract format
+        let format_value: Symbol = kwargs
+            .fetch::<_, Symbol>(ruby.to_symbol("format"))
+            .map_err(|_| {
+                Error::new(
+                    ruby.exception_arg_error(),
+                    "missing required keyword argument: format",
+                )
+            })?;
+
+        let blob_symbol = ruby.to_symbol("blob");
+        if !format_value.eql(blob_symbol)? {
+            return Err(Error::new(
+                ruby.exception_arg_error(),
+                "only :blob format is currently supported",
+            ));
+        }
+
+        // Extract output path (must be Pathname)
+        let output_value: Value =
+            kwargs
+                .fetch::<_, Value>(ruby.to_symbol("output"))
+                .map_err(|_| {
+                    Error::new(
+                        ruby.exception_arg_error(),
+                        "missing required keyword argument: output",
+                    )
+                })?;
+
+        let pathname_class: RClass = ruby.eval("Pathname")?;
+        if !output_value.is_kind_of(pathname_class) {
+            let path_class = output_value.class();
+            let class_name = unsafe { path_class.name() }.into_owned();
+            return Err(Error::new(
+                ruby.exception_type_error(),
+                format!("output must be a Pathname, got {}", class_name),
+            ));
+        }
+
+        let output_str: String = output_value.funcall("to_s", ())?;
+        let output_path = PathBuf::from(&output_str);
+
+        // Create parent directories if needed
+        if let Some(parent) = output_path.parent() {
+            std::fs::create_dir_all(parent).map_err(|e| {
+                Error::new(
+                    ruby.exception_io_error(),
+                    format!("Failed to create output directory: {}", e),
+                )
+            })?;
+        }
+
+        // Create the source data provider (downloads CLDR data)
+        let source_provider = SourceDataProvider::new();
+
+        // Create the blob exporter
+        let file = File::create(&output_path).map_err(|e| {
+            Error::new(
+                ruby.exception_io_error(),
+                format!("Failed to create output file '{}': {}", output_str, e),
+            )
+        })?;
+        let sink = BufWriter::new(file);
+
+        // Create the export driver
+        let driver = ExportDriver::new(
+            locale_families,
+            DeduplicationStrategy::Maximal.into(),
+            LocaleFallbacker::new_without_data(),
+        );
+
+        // Apply marker filter if specific markers were requested
+        let driver = match selected_markers {
+            Some(markers) => driver.with_markers(markers),
+            None => driver, // :all - export all markers
+        };
+
+        let exporter = BlobExporter::new_with_sink(Box::new(sink));
+
+        driver.export(&source_provider, exporter).map_err(|e| {
+            let error_class = helpers::get_exception_class(ruby, "ICU4X::DataGeneratorError");
+            Error::new(error_class, format!("Data export failed: {}", e))
+        })?;
+
+        Ok(())
+    }
+
+    /// Returns an array of available marker names
+    fn available_markers(ruby: &Ruby) -> Result<RArray, Error> {
+        let lookup = marker_lookup();
+        let array = ruby.ary_new();
+        // Collect unique marker names (short names only, not full type paths)
+        let mut names: Vec<&str> = lookup
+            .keys()
+            .filter(|k| !k.contains("::"))
+            .copied()
+            .collect();
+        names.sort();
+        for name in names {
+            array.push(ruby.str_new(name))?;
+        }
+        Ok(array)
+    }
+}
+
+pub fn init(ruby: &Ruby, module: &RModule) -> Result<(), Error> {
+    let class = module.define_class("DataGenerator", ruby.class_object())?;
+    class.define_singleton_method("export", function!(DataGenerator::export, 1))?;
+    class.define_singleton_method(
+        "available_markers",
+        function!(DataGenerator::available_markers, 0),
+    )?;
+    Ok(())
+}