kabosu 0.6.11.1 → 0.6.11.2.dev.20260820.2da2749

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1311b1fd889d88fc04f4bc695884b6463c64b73fa1c4166fa4bbac07af015e67
-  data.tar.gz: 8abdf827ccb81820074a6a446fc3ac735bf0931ea90b0858192bf0622510ba89
+  metadata.gz: 4a70dc4b3b987df22342c2b87fcb06cafe4aa0c3cd194ecd30fbce80f98b8281
+  data.tar.gz: 27b0027a5eddeeb89d6d3bebcd47425db2aeb309c0821c03f3e29af6f1d97b7b
 SHA512:
-  metadata.gz: 0a8f9d2d3977a0710dbc184aa2eda9cffb85b29dada72ee6e032dcaa71d875b844593ad0bf3b5786e87a64b8df4c5185efeb6387484ad0e4d8a1f44511630944
-  data.tar.gz: 41319ca91d7837a116bcf3846d253b6e8f0130f22bdfcfda956138274ce2be8d6ef80043d6c7014dbeff1ea1c0e1be8eb70018499be7935734ae345f9e8ad043
+  metadata.gz: cf0d834428557fd1591e6011d79ae1d2cb963d4fe389c8b706c3f3cc86a169a8831ef3a7e50f12ac3d01c49345a2e35ae564ddf9b73d411d039824014bbf5b30
+  data.tar.gz: 8bedb4672df3399b4f8074626e3896894fb9fba9b879f1cfd9970377265ffbf74a0aef6fd6e14687829dc945ab35270df76f0856496c36256ad813c5b3c47b90

data/ext/kabosu/src/morpheme.rs

@@ -1,5 +1,5 @@
-use magnus::value::ReprValue;
-use magnus::{gc, Error, RArray, RString, Ruby};
+use magnus::value::{Opaque, ReprValue};
+use magnus::{gc, DataTypeFunctions, Error, RArray, RString, Ruby, TypedData};
 use std::cell::OnceCell;
 use std::collections::HashMap;
 use std::sync::{Arc, Mutex, OnceLock};
@@ -75,9 +75,28 @@ pub(crate) fn rb_morpheme_from_data(
         dict,
         debug,
         word_fields: OnceCell::new(),
+        surface_cache: OnceCell::new(),
+        dictionary_form_cache: OnceCell::new(),
+        pos_cache: OnceCell::new(),
     }
 }
 
+// One frozen Ruby String per slot, built on first ask.
+//
+// FROZEN ON PURPOSE, and not only for safety: `+str` returns self for a mutable
+// String and a mutable COPY for a frozen one, so a shared mutable value would let
+// an innocent `buf = +m.surface; buf << other` write straight through into the
+// cache and corrupt every later read. Frozen, that idiom keeps working unchanged
+// and anything else fails loudly instead of silently.
+fn frozen_string(ruby: &Ruby, cell: &OnceCell<Opaque<RString>>, source: &str) -> RString {
+    let cached = cell.get_or_init(|| {
+        let rstr = ruby.str_new(source);
+        rstr.as_value().freeze();
+        Opaque::from(rstr)
+    });
+    ruby.get_inner(*cached)
+}
+
 fn vec_u32_to_array(ids: &[u32]) -> Result<RArray, Error> {
     let ruby = Ruby::get().unwrap();
     let ary = ruby.ary_new_capa(ids.len());
@@ -87,12 +106,35 @@ fn vec_u32_to_array(ids: &[u32]) -> Result<RArray, Error> {
     Ok(ary)
 }
 
-#[magnus::wrap(class = "Kabosu::Morpheme")]
+#[derive(TypedData)]
+#[magnus(class = "Kabosu::Morpheme", mark)]
 pub(crate) struct RbMorpheme {
     data: MorphemeData,
     dict: Arc<JapaneseDictionary>,
     debug: bool,
     word_fields: OnceCell<LazyWordFields>,
+    // Ruby objects handed to the same caller over and over: built once per
+    // morpheme rather than once per call. See `surface` for the reasoning and
+    // for why they are frozen.
+    surface_cache: OnceCell<Opaque<RString>>,
+    dictionary_form_cache: OnceCell<Opaque<RString>>,
+    pos_cache: OnceCell<Opaque<RArray>>,
+}
+
+impl DataTypeFunctions for RbMorpheme {
+    // The cached surface is reachable only from Rust, so the GC cannot see it
+    // without being told. Missing this would let a live String be collected.
+    fn mark(&self, marker: &gc::Marker) {
+        if let Some(cached) = self.surface_cache.get() {
+            marker.mark(*cached);
+        }
+        if let Some(cached) = self.dictionary_form_cache.get() {
+            marker.mark(*cached);
+        }
+        if let Some(cached) = self.pos_cache.get() {
+            marker.mark(*cached);
+        }
+    }
 }
 
 struct LazyWordFields {
@@ -245,11 +287,47 @@ impl RbMorpheme {
         Ok(result)
     }
 
-    pub(crate) fn surface(&self) -> &str {
-        &self.data.surface
+    // One frozen Ruby String per morpheme, not one per call.
+    //
+    // Callers ask this a lot: measured on a real tokenizer workload, 58,874 calls
+    // across 3,569 distinct morphemes, 16.5 each, so 94% of the Strings this used
+    // to allocate were identical copies of one already made. Returning `&str` let
+    // magnus build a fresh RString every time.
+    //
+    // FROZEN ON PURPOSE, and not only for safety: `+str` returns self for a
+    // mutable String and a mutable COPY for a frozen one, so a shared mutable
+    // surface would let an innocent `buf = +m.surface; buf << other` write through
+    // into the cache and corrupt every later read. Frozen, that idiom keeps
+    // working unchanged and anything else fails loudly. `part_of_speech` freezes
+    // what it caches for the same reason.
+    // Takes `&Ruby` as its first parameter rather than calling `Ruby::get()`.
+    // magnus hands that handle over for free (`Ruby::get_with` is
+    // `Self(PhantomData)`, zero instructions), where `Ruby::get()` costs a
+    // thread-local access, a RefCell borrow and a Result construction on EVERY
+    // call — including the cache-hit path, whose only use for the handle is
+    // `get_inner`, which discards it. With the allocations gone that round trip
+    // was a real share of what these accessors still cost. `method!(.., 0)` in
+    // lib.rs is unchanged: magnus resolves the &Ruby form on the fn pointer.
+    pub(crate) fn surface(ruby: &Ruby, rb_self: &Self) -> RString {
+        frozen_string(ruby, &rb_self.surface_cache, &rb_self.data.surface)
+    }
+
+    // Asked more than any other method on this class: 35.7 times per morpheme on
+    // a real workload. The process-wide POS_CACHE below already makes the ANSWER
+    // free of allocation, but every one of those calls still took a global Mutex
+    // and hashed a key to reach it. A morpheme's part of speech cannot change, so
+    // the first ask is remembered here and the rest never reach the lock.
+    pub(crate) fn part_of_speech(ruby: &Ruby, rb_self: &Self) -> Result<RArray, Error> {
+        if let Some(cached) = rb_self.pos_cache.get() {
+            return Ok(ruby.get_inner(*cached));
+        }
+
+        let ary = rb_self.pos_from_shared_cache()?;
+        let _ = rb_self.pos_cache.set(Opaque::from(ary));
+        Ok(ary)
     }
 
-    pub(crate) fn part_of_speech(&self) -> Result<RArray, Error> {
+    fn pos_from_shared_cache(&self) -> Result<RArray, Error> {
         let dict_ptr = Arc::as_ptr(&self.dict) as usize;
         let pos_id = self.data.pos_id;
 
@@ -288,8 +366,11 @@ impl RbMorpheme {
         self.data.pos_id
     }
 
-    pub(crate) fn dictionary_form(&self) -> &str {
-        &self.data.dictionary_form
+    // Cached like `surface`, and for the same measured reason: 6.0 calls per
+    // morpheme on a real workload, so five of every six Strings this built were
+    // a copy of one it had already built.
+    pub(crate) fn dictionary_form(ruby: &Ruby, rb_self: &Self) -> RString {
+        frozen_string(ruby, &rb_self.dictionary_form_cache, &rb_self.data.dictionary_form)
     }
 
     pub(crate) fn normalized_form(&self) -> &str {

data/lib/kabosu/dict_manager.rb

@@ -46,13 +46,32 @@ module Kabosu
         return dic_path
       end
 
-      url = release_asset_url(version, edition)
-      zip_path = File.join(@dir, "sudachi-dictionary-#{version}-#{edition}.zip")
-
       FileUtils.mkdir_p(@dir)
-      download(url, zip_path)
-      extract(zip_path, @dir)
-      FileUtils.rm_f(zip_path)
+
+      # SudachiDict switched to Python-only release assets in v20260723:
+      # https://github.com/WorksApplications/SudachiDict/releases/tag/v20260723
+      # The legacy `sudachi-dictionary-{version}-{edition}.zip` is gone; the
+      # new releases ship `sudachidict_{edition}-{version}-py3-none-any.whl`
+      # (a PEP 427 wheel). Pick whichever exists, preferring the legacy zip
+      # when both are present (older releases kept both formats in flight).
+      sources = pick_release_sources(version, edition)
+      raise DownloadError, "No downloadable assets for #{version}/#{edition}" if sources.empty?
+
+      sources.each do |source|
+        begin
+          download(source.fetch(:url), source.fetch(:archive_path))
+          extract(source.fetch(:archive_path), dest_dir, edition: edition)
+          FileUtils.rm_f(source.fetch(:archive_path))
+          break
+        rescue DownloadError => e
+          FileUtils.rm_f(source.fetch(:archive_path)) if File.exist?(source.fetch(:archive_path))
+          # Try the next candidate (e.g. wheel if the zip 404s). If this was
+          # the last one, surface the original error.
+          raise if sources.last.equal?(source)
+
+          warn "  falling back: #{e.message}"
+        end
+      end
 
       raise DownloadError, "Expected #{dic_path} after extraction, but file not found" unless File.exist?(dic_path)
 
@@ -181,6 +200,51 @@ module Kabosu
       edition
     end
 
+    # Look up the GitHub release for `version` and return an ordered list of
+    # candidate download sources. The legacy `sudachi-dictionary-{version}-{edition}.zip`
+    # is preferred when present (matches the layout every other consumer
+    # assumes); otherwise we fall back to the SudachiDict Python wheel,
+    # which is what v20260723+ ship and which carries `system.dic` inside
+    # `sudachidict_{edition}/resources/`.
+    #
+    # On GitHub API failure we degrade to the legacy zip URL alone: an old
+    # release that 404s on the zip *also* 404s on the wheel, so falling back
+    # to "guessed" candidates costs nothing and keeps the code path simple
+    # for offline / no-network callers.
+    def pick_release_sources(version, edition)
+      candidates = []
+      zip = release_asset_url(version, edition)
+      candidates << { url: zip, archive_path: File.join(@dir, "sudachi-dictionary-#{version}-#{edition}.zip"), format: :zip }
+
+      begin
+        release = fetch_release(version)
+        if release
+          wheel_name = "sudachidict_#{edition}-#{version}-py3-none-any.whl"
+          wheel = release["assets"].to_a.find { |a| a["name"] == wheel_name }
+          if wheel && wheel["browser_download_url"]
+            candidates << {
+              url: wheel.fetch("browser_download_url"),
+              archive_path: File.join(@dir, wheel_name),
+              format: :wheel
+            }
+          end
+        end
+      rescue DownloadError
+        # API failure is fine: keep the legacy-zip candidate; the user will
+        # see a clean 404 if neither asset actually exists on disk.
+      end
+
+      candidates
+    end
+
+    def fetch_release(version)
+      uri = URI("#{GITHUB_API}/repos/#{GITHUB_REPO}/releases/tags/v#{version}")
+      response = http_get(uri, headers: { "Accept" => "application/json" })
+      return nil unless response.is_a?(Net::HTTPSuccess)
+
+      JSON.parse(response.body)
+    end
+
     def release_asset_url(version, edition)
       "https://github.com/#{GITHUB_REPO}/releases/download/v#{version}/sudachi-dictionary-#{version}-#{edition}.zip"
     end
@@ -249,11 +313,22 @@ module Kabosu
       end
     end
 
-    def extract(zip_path, dest_dir)
+    def extract(zip_path, dest_dir, edition: nil)
       warn "Extracting..."
       Zip::File.open(zip_path) do |archive|
         archive.each do |entry|
-          target = File.join(dest_dir, entry.name)
+          # Wheels (`sudachidict_{edition}/resources/system.dic`) carry the
+          # edition in the top-level directory; legacy zips name the file
+          # `system_{edition}.dic` already. When the entry is a `system.dic`
+          # inside a wheel and `edition:` was passed, land it at
+          # `dest_dir/system_{edition}.dic` so `find`/`installed` keep working
+          # without a separate code path.
+          target_name = entry.name
+          if edition && File.basename(entry.name) == "system.dic"
+            target_name = "system_#{edition}.dic"
+          end
+
+          target = File.join(dest_dir, target_name)
           # Guard against zip-slip — refuse entries that escape dest_dir.
           unless File.expand_path(target).start_with?(File.expand_path(dest_dir) + File::SEPARATOR)
             raise DownloadError, "Refusing to extract entry outside dest_dir: #{entry.name}"

data/lib/kabosu/version.rb

@@ -1,3 +1,3 @@
 module Kabosu
-  VERSION = "0.6.11.1".freeze
+  VERSION = "0.6.11.2.dev.20260820.2da2749".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kabosu
 version: !ruby/object:Gem::Version
-  version: 0.6.11.1
+  version: 0.6.11.2.dev.20260820.2da2749
 platform: ruby
 authors:
 - davafons
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-06-30 00:00:00.000000000 Z
+date: 2026-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys