kabosu 0.6.10.1.dev.20260531.8cc8132 → 0.6.10.1.dev.20260531.a468755

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fd256d0abf1e3e9070c8282ca30378858d06d73a2379586f2205158cdb3c7b40
-  data.tar.gz: 52606c444dbc697639fd15318842b0d226288f002b3bc339ffb42b4094069952
+  metadata.gz: b4ab39bb0ce7aacedbd9640abb955beecd3e11f5ba1da56505efa7eccd45afb3
+  data.tar.gz: 69cabebc7ae242d7222408929f81613e1e89a431d25b79586593d81170eb83d9
 SHA512:
-  metadata.gz: efeabfb7bf03ccda85fad30a0a12973aec341d8d1d5a775778552c13ebb24d0e8111e6aeede41e920701eef1cae7704ccdc760c322f1cd9a07f451dca54e4a73
-  data.tar.gz: c51bcaf3687ab4ca9b03bc2637a360207a9afe32309787c4ea28da5e1e32e45fe2f836937a17261559be67c175b1176ff8dcf25ce818d4d367e1493aac6556d7
+  metadata.gz: 7eaa31344e593da582f9ce264b500ec4360b38294a93813002387f03ce153ee555e53fa49f824ead5fdbabe4efc4a66c12699b1721d4413dbb3e1b3c5d85c794
+  data.tar.gz: 04a175af4642fc5757eb1bf7df34e9bf875811a22df7263e7f4b226537017705f9777e774fa81520652c4a671ca79d65d2dfa210d3a76a11127c040def6bdc2d

data/ext/kabosu/src/lib.rs

@@ -1,6 +1,5 @@
 mod dictionary;
 mod errors;
-mod grouping;
 mod morpheme;
 mod nogvl;
 mod parsing;
@@ -39,7 +38,6 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     batch_class.define_method("internal_cost", method!(RbTokenBatch::internal_cost, 0))?;
     batch_class.define_method("morpheme_at", method!(RbTokenBatch::morpheme_at, 1))?;
     batch_class.define_method("surfaces", method!(RbTokenBatch::surfaces, 0))?;
-    batch_class.define_method("group_morphemes", method!(RbTokenBatch::group_morphemes, 0))?;
 
     // Kabosu::Tokenizer
     let tok_class = module.define_class("Tokenizer", ruby.class_object())?;

data/ext/kabosu/src/token_batch.rs

@@ -2,7 +2,6 @@ use magnus::{Error, RArray, Ruby};
 use std::sync::Arc;
 use sudachi::dic::dictionary::JapaneseDictionary;
 
-use crate::grouping::group_morphemes_rust;
 use crate::morpheme::{rb_morpheme_from_data, MorphemeData, RbMorpheme};
 
 #[magnus::wrap(class = "Kabosu::TokenBatch")]
@@ -51,10 +50,4 @@ impl RbTokenBatch {
         }
         Ok(ary)
     }
-
-    /// jpdb-style grouping performed natively in Rust.
-    /// Returns an Array-of-Array of Kabosu::Morpheme.
-    pub(crate) fn group_morphemes(&self) -> Result<RArray, Error> {
-        group_morphemes_rust(&self.morphemes, &self.dict, self.debug)
-    }
 }

data/lib/kabosu/morpheme_list.rb

@@ -92,54 +92,6 @@ module Kabosu
       surfaces.join
     end
 
-    # jpdb-style grouping performed natively in Rust when backed by a lazy
-    # source. Falls back to a Ruby implementation for already-materialized
-    # lists so the method is always safe to call.
-    def group_morphemes
-      if @source&.respond_to?(:group_morphemes)
-        return @source.group_morphemes
-      end
-
-      groups = []
-      each do |m|
-        last = groups.last
-        if last && content_word?(last.first) && extends_group?(m, last.last)
-          last << m
-        else
-          groups << [m]
-        end
-      end
-      groups
-    end
-
-    private
-
-    def content_word?(morpheme)
-      !%w[助詞 助動詞 補助記号 記号 空白].include?(morpheme.part_of_speech.first)
-    end
-
-    def extends_group?(morpheme, prev = nil)
-      pos = morpheme.part_of_speech
-      pos1 = pos[0]
-      pos1 == "助動詞" ||
-        (pos1 == "助詞" && !clause_boundary?(morpheme) &&
-         (pos[1] == "接続助詞" ||
-          (pos[1] == "副助詞" && prev && %w[動詞 形容詞 形状詞].include?(prev.part_of_speech[0])))) ||
-        (pos1 == "動詞" && pos[1] == "非自立可能" &&
-         prev && prev.part_of_speech[0] == "助詞" && %w[て で].include?(prev.surface))
-    end
-
-    def clause_boundary?(morpheme)
-      return false unless morpheme
-      pos = morpheme.part_of_speech
-      return true if pos[0] == "助詞" &&
-                     %w[ながら たら ば と のに から ので けれど けど つつ なり や か かどうか とも].include?(morpheme.surface)
-      return true if pos[0] == "助詞" && pos[1] == "接続助詞" && morpheme.surface == "が"
-      false
-    end
-
-    public
-
     # Filter morphemes by POS. Accepts a PosMatcher or an array pattern.
     # Returns a new MorphemeList with only matching morphemes.
     #

data/lib/kabosu/version.rb

@@ -1,3 +1,3 @@
 module Kabosu
-  VERSION = "0.6.10.1.dev.20260531.8cc8132"
+  VERSION = "0.6.10.1.dev.20260531.a468755"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kabosu
 version: !ruby/object:Gem::Version
-  version: 0.6.10.1.dev.20260531.8cc8132
+  version: 0.6.10.1.dev.20260531.a468755
 platform: ruby
 authors:
 - davafons
@@ -110,7 +110,6 @@ files:
 - ext/kabosu/extconf.rb
 - ext/kabosu/src/dictionary.rs
 - ext/kabosu/src/errors.rs
-- ext/kabosu/src/grouping.rs
 - ext/kabosu/src/lib.rs
 - ext/kabosu/src/morpheme.rs
 - ext/kabosu/src/nogvl.rs

data/ext/kabosu/src/grouping.rs

@@ -1,133 +0,0 @@
-use magnus::{Error, RArray, Ruby};
-use std::sync::Arc;
-use sudachi::dic::dictionary::JapaneseDictionary;
-
-use crate::morpheme::{rb_morpheme_from_data, MorphemeData};
-
-/// Group raw `MorphemeData` into jpdb-style chips.
-pub(crate) fn group_morphemes_rust(
-    morphemes: &[MorphemeData],
-    dict: &Arc<JapaneseDictionary>,
-    debug: bool,
-) -> Result<RArray, Error> {
-    let ruby = Ruby::get().unwrap();
-    let mut groups: Vec<Vec<MorphemeData>> = Vec::with_capacity(morphemes.len());
-
-    for m in morphemes {
-        if let Some(last_group) = groups.last_mut() {
-            let head = &last_group[0];
-            let prev = last_group.last().unwrap();
-            if is_content_word(head.pos_id, dict) && extends_group(m, prev, dict) {
-                last_group.push(m.clone());
-                continue;
-            }
-        }
-        groups.push(vec![m.clone()]);
-    }
-
-    let result = ruby.ary_new();
-    for group in groups {
-        let group_ary = ruby.ary_new();
-        for data in group {
-            group_ary.push(rb_morpheme_from_data(data, dict.clone(), debug))?;
-        }
-        result.push(group_ary)?;
-    }
-    Ok(result)
-}
-
-// POS helpers
-
-fn is_content_word(pos_id: u16, dict: &JapaneseDictionary) -> bool {
-    match dict.grammar().pos_components(pos_id).first().map(|s| s.as_str()) {
-        Some("助詞") | Some("助動詞") | Some("補助記号") | Some("記号") | Some("空白") => false,
-        _ => true,
-    }
-}
-
-fn extends_group(
-    m: &MorphemeData,
-    prev: &MorphemeData,
-    dict: &JapaneseDictionary,
-) -> bool {
-    let comps = dict.grammar().pos_components(m.pos_id);
-    let pos0 = comps.first().map(|s| s.as_str());
-    let pos1 = comps.get(1).map(|s| s.as_str());
-
-    match pos0 {
-        Some("助動詞") => true,
-        Some("助詞") => {
-            if is_clause_boundary(m.surface.as_str(), m.pos_id, dict) {
-                return false;
-            }
-            if pos1 == Some("接続助詞") {
-                return true;
-            }
-            // 副助詞 clings to preceding verb/adjective (e.g. たり/だり)
-            if pos1 == Some("副助詞") && is_verb_adj_adv(prev.pos_id, dict) {
-                return true;
-            }
-            false
-        }
-        Some("動詞") => {
-            if pos1 != Some("非自立可能") {
-                return false;
-            }
-            // te-form auxiliary chain: て/で + いる/ある/くる/etc.
-            let prev_pos0 = dict.grammar().pos_components(prev.pos_id).first().map(|s| s.as_str());
-            if prev_pos0 == Some("助詞")
-                && (prev.surface == "て" || prev.surface == "で")
-            {
-                return true;
-            }
-            // compound verb (V+V) intentionally skipped — caller handles DB lookup
-            false
-        }
-        _ => false,
-    }
-}
-
-fn is_clause_boundary(surface: &str, pos_id: u16, dict: &JapaneseDictionary) -> bool {
-    let comps = dict.grammar().pos_components(pos_id);
-    let pos0 = comps.first().map(|s| s.as_str());
-    let pos1 = comps.get(1).map(|s| s.as_str());
-
-    if pos0 == Some("助詞") {
-        if is_clause_boundary_particle(surface) {
-            return true;
-        }
-        // contrastive が (接続助詞) is a boundary, unlike subject が (格助詞)
-        if pos1 == Some("接続助詞") && surface == "が" {
-            return true;
-        }
-    }
-    false
-}
-
-fn is_clause_boundary_particle(surface: &str) -> bool {
-    matches!(
-        surface,
-        "ながら"
-            | "たら"
-            | "ば"
-            | "と"
-            | "のに"
-            | "から"
-            | "ので"
-            | "けれど"
-            | "けど"
-            | "つつ"
-            | "なり"
-            | "や"
-            | "か"
-            | "かどうか"
-            | "とも"
-    )
-}
-
-fn is_verb_adj_adv(pos_id: u16, dict: &JapaneseDictionary) -> bool {
-    matches!(
-        dict.grammar().pos_components(pos_id).first().map(|s| s.as_str()),
-        Some("動詞") | Some("形容詞") | Some("形状詞")
-    )
-}