kabosu 0.6.10.1 → 0.6.10.2.dev.20260531.b95a56a

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 82efeb6b51fcfc69c65004ffbd3bbc63a19c6c78b3a09569cdd8329e72f4766b
-  data.tar.gz: b282e3c3ad1b723c7a6aef61a46748c601d148af90639632be00a790567700b9
+  metadata.gz: 3ba5a31baf20e7e4958a21f14d0b9194a9b83292b2823192bfe988c7bcb3b121
+  data.tar.gz: efbd5e6664154257c9ce993d85f2c06e11376f70e444102eb75435210f64def7
 SHA512:
-  metadata.gz: 310ce82c04d9e735fd6df5cb70e36ccdc93a54f290daa85d86a296bbe4106cc94d4b60ec76bd0a486769c5d2f2d9c17e4bd0b99b13cba602341a8c43bf5b8f33
-  data.tar.gz: 180d32c31053365e2360b2d2b403e1b33fbeb78cbf6bdc1fc3767d4071507f37c33898c52e00d9cf0ef694ce94256fc7df906b026da0c5a0c0feb6f7514d1fe5
+  metadata.gz: d6bfc7e8d82719596d5b25057983136e2ba4f9f6548021c964595d2d3f69bfdee56faf77c2ee67186bad0a4e3f6a08b6f9c13743ebfaaf81f415fd65dbd70b10
+  data.tar.gz: '0929bb3dcd92ddbf4bb3d6e7ba7969d9661ad0421aef553a80b2da9560f438401a430ef0f448bf6f7db30464cdee6f0b57444fce9816c2a2a0abda8b718a1175'

data/ext/kabosu/src/grouping.rs

@@ -0,0 +1,133 @@
+use magnus::{Error, RArray, Ruby};
+use std::sync::Arc;
+use sudachi::dic::dictionary::JapaneseDictionary;
+
+use crate::morpheme::{rb_morpheme_from_data, MorphemeData};
+
+/// Group raw `MorphemeData` into jpdb-style chips.
+pub(crate) fn group_morphemes_rust(
+    morphemes: &[MorphemeData],
+    dict: &Arc<JapaneseDictionary>,
+    debug: bool,
+) -> Result<RArray, Error> {
+    let ruby = Ruby::get().unwrap();
+    let mut groups: Vec<Vec<MorphemeData>> = Vec::with_capacity(morphemes.len());
+
+    for m in morphemes {
+        if let Some(last_group) = groups.last_mut() {
+            let head = &last_group[0];
+            let prev = last_group.last().unwrap();
+            if is_content_word(head.pos_id, dict) && extends_group(m, prev, dict) {
+                last_group.push(m.clone());
+                continue;
+            }
+        }
+        groups.push(vec![m.clone()]);
+    }
+
+    let result = ruby.ary_new();
+    for group in groups {
+        let group_ary = ruby.ary_new();
+        for data in group {
+            group_ary.push(rb_morpheme_from_data(data, dict.clone(), debug))?;
+        }
+        result.push(group_ary)?;
+    }
+    Ok(result)
+}
+
+// POS helpers
+
+fn is_content_word(pos_id: u16, dict: &JapaneseDictionary) -> bool {
+    match dict.grammar().pos_components(pos_id).first().map(|s| s.as_str()) {
+        Some("助詞") | Some("助動詞") | Some("補助記号") | Some("記号") | Some("空白") => false,
+        _ => true,
+    }
+}
+
+fn extends_group(
+    m: &MorphemeData,
+    prev: &MorphemeData,
+    dict: &JapaneseDictionary,
+) -> bool {
+    let comps = dict.grammar().pos_components(m.pos_id);
+    let pos0 = comps.first().map(|s| s.as_str());
+    let pos1 = comps.get(1).map(|s| s.as_str());
+
+    match pos0 {
+        Some("助動詞") => true,
+        Some("助詞") => {
+            if is_clause_boundary(m.surface.as_str(), m.pos_id, dict) {
+                return false;
+            }
+            if pos1 == Some("接続助詞") {
+                return true;
+            }
+            // 副助詞 clings to preceding verb/adjective (e.g. たり/だり)
+            if pos1 == Some("副助詞") && is_verb_adj_adv(prev.pos_id, dict) {
+                return true;
+            }
+            false
+        }
+        Some("動詞") => {
+            if pos1 != Some("非自立可能") {
+                return false;
+            }
+            // te-form auxiliary chain: て/で + いる/ある/くる/etc.
+            let prev_pos0 = dict.grammar().pos_components(prev.pos_id).first().map(|s| s.as_str());
+            if prev_pos0 == Some("助詞")
+                && (prev.surface == "て" || prev.surface == "で")
+            {
+                return true;
+            }
+            // compound verb (V+V) intentionally skipped — caller handles DB lookup
+            false
+        }
+        _ => false,
+    }
+}
+
+fn is_clause_boundary(surface: &str, pos_id: u16, dict: &JapaneseDictionary) -> bool {
+    let comps = dict.grammar().pos_components(pos_id);
+    let pos0 = comps.first().map(|s| s.as_str());
+    let pos1 = comps.get(1).map(|s| s.as_str());
+
+    if pos0 == Some("助詞") {
+        if is_clause_boundary_particle(surface) {
+            return true;
+        }
+        // contrastive が (接続助詞) is a boundary, unlike subject が (格助詞)
+        if pos1 == Some("接続助詞") && surface == "が" {
+            return true;
+        }
+    }
+    false
+}
+
+fn is_clause_boundary_particle(surface: &str) -> bool {
+    matches!(
+        surface,
+        "ながら"
+            | "たら"
+            | "ば"
+            | "と"
+            | "のに"
+            | "から"
+            | "ので"
+            | "けれど"
+            | "けど"
+            | "つつ"
+            | "なり"
+            | "や"
+            | "か"
+            | "かどうか"
+            | "とも"
+    )
+}
+
+fn is_verb_adj_adv(pos_id: u16, dict: &JapaneseDictionary) -> bool {
+    matches!(
+        dict.grammar().pos_components(pos_id).first().map(|s| s.as_str()),
+        Some("動詞") | Some("形容詞") | Some("形状詞")
+    )
+}

data/ext/kabosu/src/lib.rs

@@ -1,5 +1,6 @@
 mod dictionary;
 mod errors;
+mod grouping;
 mod morpheme;
 mod nogvl;
 mod parsing;
@@ -38,6 +39,7 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     batch_class.define_method("internal_cost", method!(RbTokenBatch::internal_cost, 0))?;
     batch_class.define_method("morpheme_at", method!(RbTokenBatch::morpheme_at, 1))?;
     batch_class.define_method("surfaces", method!(RbTokenBatch::surfaces, 0))?;
+    batch_class.define_method("group_morphemes", method!(RbTokenBatch::group_morphemes, 0))?;
 
     // Kabosu::Tokenizer
     let tok_class = module.define_class("Tokenizer", ruby.class_object())?;

data/ext/kabosu/src/morpheme.rs

@@ -1,6 +1,8 @@
-use magnus::{Error, RArray, Ruby};
+use magnus::value::ReprValue;
+use magnus::{gc, Error, RArray, RString, Ruby};
 use std::cell::OnceCell;
-use std::sync::Arc;
+use std::collections::HashMap;
+use std::sync::{Arc, Mutex, OnceLock};
 use sudachi::analysis::morpheme::Morpheme as SudachiMorpheme;
 use sudachi::analysis::stateless_tokenizer::DictionaryAccess;
 use sudachi::analysis::Mode;
@@ -57,6 +59,12 @@ where
     }
 }
 
+#[derive(Clone, Copy)]
+struct CachedRArray(RArray);
+unsafe impl Send for CachedRArray {}
+
+static POS_CACHE: OnceLock<Mutex<HashMap<(usize, u16), CachedRArray>>> = OnceLock::new();
+
 pub(crate) fn rb_morpheme_from_data(
     data: MorphemeData,
     dict: Arc<JapaneseDictionary>,
@@ -66,7 +74,6 @@ pub(crate) fn rb_morpheme_from_data(
         data,
         dict,
         debug,
-        pos: OnceCell::new(),
         word_fields: OnceCell::new(),
     }
 }
@@ -85,7 +92,6 @@ pub(crate) struct RbMorpheme {
     data: MorphemeData,
     dict: Arc<JapaneseDictionary>,
     debug: bool,
-    pos: OnceCell<Vec<String>>,
     word_fields: OnceCell<LazyWordFields>,
 }
 
@@ -243,17 +249,32 @@ impl RbMorpheme {
         &self.data.surface
     }
 
-    pub(crate) fn part_of_speech(&self) -> Vec<String> {
-        self.pos
-            .get_or_init(|| {
-                self.dict
-                    .grammar()
-                    .pos_components(self.data.pos_id)
-                    .iter()
-                    .map(|s| s.to_string())
-                    .collect()
-            })
-            .clone()
+    pub(crate) fn part_of_speech(&self) -> Result<RArray, Error> {
+        let dict_ptr = Arc::as_ptr(&self.dict) as usize;
+        let pos_id = self.data.pos_id;
+
+        {
+            let cache = POS_CACHE.get_or_init(|| Mutex::new(HashMap::new())).lock().unwrap();
+            if let Some(&cached) = cache.get(&(dict_ptr, pos_id)) {
+                return Ok(cached.0);
+            }
+        }
+
+        let ruby = Ruby::get().unwrap();
+        let ary = ruby.ary_new();
+        for s in self.dict.grammar().pos_components(pos_id) {
+            let rstr: RString = ruby.str_new(s);
+            rstr.as_value().freeze();
+            ary.push(rstr)?;
+        }
+        ary.as_value().freeze();
+        // Pin permanently: POS combinations are bounded (~hundreds) and frozen
+        // arrays are tiny. The cache lives for the process lifetime.
+        gc::register_mark_object(ary);
+
+        POS_CACHE.get_or_init(|| Mutex::new(HashMap::new())).lock().unwrap().insert((dict_ptr, pos_id), CachedRArray(ary));
+
+        Ok(ary)
     }
 
     pub(crate) fn part_of_speech_id(&self) -> u16 {

data/ext/kabosu/src/token_batch.rs

@@ -2,6 +2,7 @@ use magnus::{Error, RArray, Ruby};
 use std::sync::Arc;
 use sudachi::dic::dictionary::JapaneseDictionary;
 
+use crate::grouping::group_morphemes_rust;
 use crate::morpheme::{rb_morpheme_from_data, MorphemeData, RbMorpheme};
 
 #[magnus::wrap(class = "Kabosu::TokenBatch")]
@@ -50,4 +51,10 @@ impl RbTokenBatch {
         }
         Ok(ary)
     }
+
+    /// jpdb-style grouping performed natively in Rust.
+    /// Returns an Array-of-Array of Kabosu::Morpheme.
+    pub(crate) fn group_morphemes(&self) -> Result<RArray, Error> {
+        group_morphemes_rust(&self.morphemes, &self.dict, self.debug)
+    }
 }

data/lib/kabosu/morpheme_list.rb

@@ -92,6 +92,54 @@ module Kabosu
       surfaces.join
     end
 
+    # jpdb-style grouping performed natively in Rust when backed by a lazy
+    # source. Falls back to a Ruby implementation for already-materialized
+    # lists so the method is always safe to call.
+    def group_morphemes
+      if @source&.respond_to?(:group_morphemes)
+        return @source.group_morphemes
+      end
+
+      groups = []
+      each do |m|
+        last = groups.last
+        if last && content_word?(last.first) && extends_group?(m, last.last)
+          last << m
+        else
+          groups << [m]
+        end
+      end
+      groups
+    end
+
+    private
+
+    def content_word?(morpheme)
+      !%w[助詞 助動詞 補助記号 記号 空白].include?(morpheme.part_of_speech.first)
+    end
+
+    def extends_group?(morpheme, prev = nil)
+      pos = morpheme.part_of_speech
+      pos1 = pos[0]
+      pos1 == "助動詞" ||
+        (pos1 == "助詞" && !clause_boundary?(morpheme) &&
+         (pos[1] == "接続助詞" ||
+          (pos[1] == "副助詞" && prev && %w[動詞 形容詞 形状詞].include?(prev.part_of_speech[0])))) ||
+        (pos1 == "動詞" && pos[1] == "非自立可能" &&
+         prev && prev.part_of_speech[0] == "助詞" && %w[て で].include?(prev.surface))
+    end
+
+    def clause_boundary?(morpheme)
+      return false unless morpheme
+      pos = morpheme.part_of_speech
+      return true if pos[0] == "助詞" &&
+                     %w[ながら たら ば と のに から ので けれど けど つつ なり や か かどうか とも].include?(morpheme.surface)
+      return true if pos[0] == "助詞" && pos[1] == "接続助詞" && morpheme.surface == "が"
+      false
+    end
+
+    public
+
     # Filter morphemes by POS. Accepts a PosMatcher or an array pattern.
     # Returns a new MorphemeList with only matching morphemes.
     #

data/lib/kabosu/version.rb

@@ -1,3 +1,3 @@
 module Kabosu
-  VERSION = "0.6.10.1"
+  VERSION = "0.6.10.2.dev.20260531.b95a56a"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kabosu
 version: !ruby/object:Gem::Version
-  version: 0.6.10.1
+  version: 0.6.10.2.dev.20260531.b95a56a
 platform: ruby
 authors:
 - davafons
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-04-30 00:00:00.000000000 Z
+date: 2026-05-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -110,6 +110,7 @@ files:
 - ext/kabosu/extconf.rb
 - ext/kabosu/src/dictionary.rs
 - ext/kabosu/src/errors.rs
+- ext/kabosu/src/grouping.rs
 - ext/kabosu/src/lib.rs
 - ext/kabosu/src/morpheme.rs
 - ext/kabosu/src/nogvl.rs