lindera 6.1.0 → 6.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 626630dcc1ced79783abf165ad0143ecb945f7ffe96365a09ceeba51a543303c
-  data.tar.gz: 03dcddc789f4aca2a0c46a7371b16d1554f4b81c01070fbe2717363146ab7afb
+  metadata.gz: 529da66cbb1d0fd3372e0ab7695c72ac53c55d8a1a411b99d3db0135ac05be56
+  data.tar.gz: '020678eec7845a566b1572f2a61af447252dc0090a5d9c73f183cddf4e656f46'
 SHA512:
-  metadata.gz: fb0a91609265bbe897ce62edc052cea47621f5b5354cc6a6a184dc425f19ba9019c34b5e5af81811e00e96848deb52b59773b4a2f429f230de7a63961db837e2
-  data.tar.gz: 49b309e6355434b3ccb54ad7b35777f660c3371c8195c108dc4b0bc07ec3199d0c0d11b3885cc86b97ffec38030272591817828e2fe5daa24721a86de1522613
+  metadata.gz: a42a702e498f3690ba0ccc3288d09c0d3e5dcc457905d058c101d112cfe24ecd4fda6cedb0e92a675a519880f6d5383efb4b1519fa1111b0b17ecaa5778e6f68
+  data.tar.gz: 57dcbda79af341e03428c7eb963e407e22bdf78a23687e95be022cdb3b91789c9c1a3b97d01b500ff44e8f700a4ea109b5262866b699d8966ae4f2597546f7d9

data/Cargo.toml

@@ -13,7 +13,7 @@
 edition = "2024"
 rust-version = "1.88"
 name = "lindera-ruby"
-version = "6.1.0"
+version = "6.2.0"
 build = false
 publish = false
 autolib = false
@@ -58,10 +58,10 @@ crate-type = [
 path = "src/lib.rs"
 
 [dependencies.lindera]
-version = "=6.1.0"
+version = "=6.2.0"
 
 [dependencies.lindera-binding-core]
-version = "=6.1.0"
+version = "=6.2.0"
 
 [dependencies.magnus]
 version = "0.8.2"

data/src/tokenizer.rs

@@ -9,6 +9,7 @@ use std::cell::RefCell;
 use std::path::Path;
 
 use magnus::prelude::*;
+use magnus::scan_args::{get_kwargs, scan_args};
 use magnus::{Error, RArray, RHash, Ruby, Value, function, method};
 
 use lindera_binding_core::{CoreTokenizer, CoreTokenizerBuilder};
@@ -199,28 +200,55 @@ pub struct RbTokenizer {
 
 /// Creates a new tokenizer with the given dictionary and mode.
 ///
+/// Ruby signature: `Tokenizer.new(dictionary, mode = nil, user_dictionary = nil,
+/// space_penalty: nil)`.
+///
 /// # Arguments
 ///
-/// * `dictionary` - Dictionary to use.
-/// * `mode` - Tokenization mode ("normal" or "decompose"). Defaults to "normal".
-/// * `user_dictionary` - Optional user dictionary.
+/// * `args` - The Ruby arguments:
+///   * `dictionary` - Dictionary to use.
+///   * `mode` - Tokenization mode ("normal" or "decompose"). `nil` or
+///     omitted means "normal".
+///   * `user_dictionary` - Optional user dictionary.
+///   * `space_penalty:` - The left-space penalty, in the forms
+///     `TokenizerBuilder#set_space_penalty` accepts. `nil` or omitted keeps
+///     the rules the dictionary ships.
 ///
 /// # Returns
 ///
-/// A new `RbTokenizer` instance.
-fn tokenizer_new(
-    dictionary: &RbDictionary,
-    mode: Option<String>,
-    user_dictionary: Option<&RbUserDictionary>,
-) -> Result<RbTokenizer, Error> {
+/// A new `RbTokenizer` instance, an `ArgumentError` for a wrong number of
+/// arguments or an unknown keyword, or the errors `set_space_penalty` and
+/// `build` raise for an invalid space-penalty setting.
+fn tokenizer_new(args: &[Value]) -> Result<RbTokenizer, Error> {
     let ruby = Ruby::get().expect("Ruby runtime not initialized");
+    let args = scan_args::<
+        (&RbDictionary,),
+        (Option<Option<String>>, Option<Option<&RbUserDictionary>>),
+        (),
+        (),
+        RHash,
+        (),
+    >(args)?;
+    let (dictionary,) = args.required;
+    let (mode, user_dictionary) = args.optional;
+    let keywords =
+        get_kwargs::<_, (), (Option<Value>,), ()>(args.keywords, &[], &["space_penalty"])?;
+    let (space_penalty,) = keywords.optional;
+
+    let mode = mode.flatten();
+    let user_dictionary = user_dictionary.flatten();
     let mode_str = mode.as_deref().unwrap_or("normal");
+    let space_penalty = match space_penalty {
+        Some(value) => rb_value_to_json(&ruby, value)?,
+        None => serde_json::Value::Null,
+    };
 
     let dict = dictionary.inner.clone();
     let user_dict = user_dictionary.map(|d| d.inner.clone());
 
-    let inner = CoreTokenizer::from_segmenter(mode_str, dict, user_dict)
-        .map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
+    let inner =
+        CoreTokenizer::from_segmenter_with_space_penalty(mode_str, dict, user_dict, &space_penalty)
+            .map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
 
     Ok(RbTokenizer { inner })
 }
@@ -361,7 +389,7 @@ pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
     builder_class.define_method("build", method!(RbTokenizerBuilder::build, 0))?;
 
     let tokenizer_class = module.define_class("Tokenizer", ruby.class_object())?;
-    tokenizer_class.define_singleton_method("new", function!(tokenizer_new, 3))?;
+    tokenizer_class.define_singleton_method("new", function!(tokenizer_new, -1))?;
     tokenizer_class.define_method("tokenize", method!(RbTokenizer::tokenize, 1))?;
     tokenizer_class.define_method(
         "tokenize_surfaces",

data/src/util.rs

@@ -3,6 +3,7 @@
 //! This module provides helper functions for converting between Ruby objects
 //! and Rust data structures, particularly for working with JSON-like data.
 
+use lindera_binding_core::argument::{MAX_ARGUMENT_DEPTH, argument_too_deep_message};
 use magnus::prelude::*;
 use magnus::{Error, RArray, RHash, Ruby, TryConvert, Value};
 
@@ -20,8 +21,26 @@ use magnus::{Error, RArray, RHash, Ruby, TryConvert, Value};
 /// # Errors
 ///
 /// Returns a `TypeError` if the Ruby value type is not supported, or if it
-/// is a non-finite `Float` (NaN or Infinity), which JSON cannot represent.
+/// is a non-finite `Float` (NaN or Infinity), which JSON cannot represent,
+/// and an `ArgumentError` if it nests arrays and hashes more than
+/// [`MAX_ARGUMENT_DEPTH`] levels deep (as one that contains itself does).
 pub fn rb_value_to_json(ruby: &Ruby, value: Value) -> Result<serde_json::Value, Error> {
+    value_at_depth(ruby, value, 0)
+}
+
+/// Converts a Ruby value that sits inside `depth` arrays and hashes.
+///
+/// # Arguments
+///
+/// * `ruby` - Ruby runtime handle.
+/// * `value` - Ruby value to convert.
+/// * `depth` - The number of arrays and hashes around `value`.
+///
+/// # Returns
+///
+/// A `serde_json::Value` representing the Ruby value, or the errors of
+/// [`rb_value_to_json`].
+fn value_at_depth(ruby: &Ruby, value: Value, depth: usize) -> Result<serde_json::Value, Error> {
     if value.is_nil() {
         Ok(serde_json::Value::Null)
     } else if value.is_kind_of(ruby.class_true_class())
@@ -68,9 +87,9 @@ pub fn rb_value_to_json(ruby: &Ruby, value: Value) -> Result<serde_json::Value,
         })?;
         Ok(serde_json::Value::String(s))
     } else if let Ok(arr) = RArray::try_convert(value) {
-        rb_array_to_json(ruby, arr)
+        array_at_depth(ruby, arr, depth)
     } else if let Ok(hash) = RHash::try_convert(value) {
-        rb_hash_to_json(ruby, hash)
+        hash_at_depth(ruby, hash, depth)
     } else {
         Err(Error::new(
             ruby.exception_type_error(),
@@ -81,20 +100,49 @@ pub fn rb_value_to_json(ruby: &Ruby, value: Value) -> Result<serde_json::Value,
     }
 }
 
-/// Converts a Ruby array to a serde_json::Value::Array.
+/// Returns the depth of an array or hash that opens inside `depth` others.
+///
+/// # Arguments
+///
+/// * `ruby` - Ruby runtime handle.
+/// * `depth` - The number of arrays and hashes around the new one.
+///
+/// # Returns
+///
+/// The new depth, or an `ArgumentError` when it exceeds
+/// [`MAX_ARGUMENT_DEPTH`].
+fn container_depth(ruby: &Ruby, depth: usize) -> Result<usize, Error> {
+    let depth = depth + 1;
+    if depth > MAX_ARGUMENT_DEPTH {
+        Err(Error::new(
+            ruby.exception_arg_error(),
+            argument_too_deep_message(),
+        ))
+    } else {
+        Ok(depth)
+    }
+}
+
+/// Converts a Ruby array that sits inside `depth` arrays and hashes.
+///
+/// The depth is counted here rather than in [`value_at_depth`] so that an
+/// array obtained through `to_ary` counts too.
 ///
 /// # Arguments
 ///
 /// * `ruby` - Ruby runtime handle.
 /// * `array` - Ruby array to convert.
+/// * `depth` - The number of arrays and hashes around `array`.
 ///
 /// # Returns
 ///
-/// A `serde_json::Value` representing the array.
-fn rb_array_to_json(ruby: &Ruby, array: RArray) -> Result<serde_json::Value, Error> {
+/// A `serde_json::Value` representing the array, or the errors of
+/// [`rb_value_to_json`].
+fn array_at_depth(ruby: &Ruby, array: RArray, depth: usize) -> Result<serde_json::Value, Error> {
+    let depth = container_depth(ruby, depth)?;
     let mut vec = Vec::new();
     for item in array.into_iter() {
-        vec.push(rb_value_to_json(ruby, item)?);
+        vec.push(value_at_depth(ruby, item, depth)?);
     }
     Ok(serde_json::Value::Array(vec))
 }
@@ -108,11 +156,32 @@ fn rb_array_to_json(ruby: &Ruby, array: RArray) -> Result<serde_json::Value, Err
 ///
 /// # Returns
 ///
-/// A `serde_json::Value` representing the hash.
+/// A `serde_json::Value` representing the hash, or the errors of
+/// [`rb_value_to_json`]; the hash itself is the first level.
 pub fn rb_hash_to_json(ruby: &Ruby, hash: RHash) -> Result<serde_json::Value, Error> {
+    hash_at_depth(ruby, hash, 0)
+}
+
+/// Converts a Ruby hash that sits inside `depth` arrays and hashes.
+///
+/// The depth is counted here rather than in [`value_at_depth`] so that a
+/// hash obtained through `to_hash` counts too.
+///
+/// # Arguments
+///
+/// * `ruby` - Ruby runtime handle.
+/// * `hash` - Ruby hash to convert.
+/// * `depth` - The number of arrays and hashes around `hash`.
+///
+/// # Returns
+///
+/// A `serde_json::Value` representing the hash, or the errors of
+/// [`rb_value_to_json`].
+fn hash_at_depth(ruby: &Ruby, hash: RHash, depth: usize) -> Result<serde_json::Value, Error> {
+    let depth = container_depth(ruby, depth)?;
     let mut map = serde_json::Map::new();
     hash.foreach(|key: String, value: Value| {
-        let json_value = rb_value_to_json(ruby, value)?;
+        let json_value = value_at_depth(ruby, value, depth)?;
         map.insert(key, json_value);
         Ok(magnus::r_hash::ForEach::Continue)
     })?;

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lindera
 version: !ruby/object:Gem::Version
-  version: 6.1.0
+  version: 6.2.0
 platform: ruby
 authors:
 - Lindera contributors