kabosu 0.6.11.4-aarch64-linux → 0.7.0.1-aarch64-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f280c805cbd2980833e2209d61058c86491e825f3effa0b0d00fe12dc1c29b36
-  data.tar.gz: 6fb1f2755f2d49ff2c6e28ba529dfe2339a21479475167b3863f07705526e913
+  metadata.gz: ba6c61184a10e59d12f7618f0769f27766422c11644ec5c47af51f7eb1775890
+  data.tar.gz: 1442e054d02e0bbfaad77d0447f32f33b00d8dec19aafd4c9f60062b0b1e11d4
 SHA512:
-  metadata.gz: a5f0e29a22aedd8414662ac4d57469e6285454d7a45d7531d509231ea9062f7eeb77f07fb3ab36c768af81565d7c40f5f2fc60c4bf21b08ba76bfab288331b84
-  data.tar.gz: a23470c82e400dba78b37c799958cf7e945ce4331d5b76999fcf28e49b895cde282326c6c6de54a91e6db9f801218f4bca942edba459e81318e46cc9dd0b8c1e
+  metadata.gz: cc59f91c5486eebb6382f68df24459d6dc34aa52c02e53197d6538c5e19830c9865a6c859535d2e95d8f082ea5beb928e0106ed6c5b7afd3a7a66220cbc847c1
+  data.tar.gz: f187ffa82f345736a0a160b4146ed6c9b6b7f4c69bfa98a03a76d80075c6670554a76063cffb3ba724d38bd3b237fdf8871a8c540eee81aa70b7168d2dfdfc46

data/README.md

@@ -43,10 +43,12 @@ morpheme.dictionary_id       # => 0                - source dictionary id
 morpheme.word_id             # => 544373           - internal word id
 morpheme.synonym_group_ids   # => []               - synonym group ids
 morpheme.dictionary_form_word_id # => -1           - dictionary-form word id
-morpheme.head_word_length    # => 3                - head word length in codepoints
+morpheme.head_word_length    # => 9                - bytes of the normalized index form
 morpheme.a_unit_split        # => [123, 456]       - split-A word ids
 morpheme.b_unit_split        # => []               - split-B word ids
 morpheme.word_structure      # => [123, 456]       - word-structure ids
+morpheme.c_unit_split        # => []               - split-C word ids (V1 dictionaries)
+morpheme.user_data           # => ""               - free text from a user dictionary's user_data column
 morpheme.total_cost          # => 5765             - morphological analysis cost
 morpheme.begin               # => 0                - start byte offset
 morpheme.end                 # => 9                - end byte offset
@@ -75,22 +77,34 @@ Then install and download a [Sudachi dictionary](https://github.com/WorksApplica
 
 ```sh
 bundle install
-bundle exec rake kabosu:install[small]  # or core, full
+bundle exec rake kabosu:install:small  # or :core, :full
 ```
 
 Dictionary editions (from smallest to largest): `small`, `core`, `full`. See the [SudachiDict documentation](https://github.com/WorksApplications/SudachiDict?tab=readme-ov-file#dictionary-types) for details on the differences between editions.
 
+kabosu 0.7 (sudachi.rs 0.7) reads only the V1 dictionary format: SudachiDict `20261015` or later.
+
+## Upgrading from 0.6
+
+sudachi.rs 0.7 changed the binary dictionary format, so a 0.6 setup needs three things:
+
+1. **Reinstall the system dictionary.** Dictionaries installed by kabosu 0.6 are V0 and cannot be loaded. Run `rake kabosu:install` (or `install_if_missing`, which ignores V0 files), then drop the old ones with `rake kabosu:remove VERSION=...`. `rake kabosu:list` marks them.
+2. **Rebuild user dictionaries** against that exact system dictionary: `rake kabosu:build_user_dict CSV=names.csv OUTPUT=names.dic` (or `Kabosu.build_user_dict`). A user dictionary now records which system dictionary it was built with, and loading it with a different one raises `Kabosu::DictionaryError`. The CSV uses the V1 column layout (see [Sudachi's user dictionary guide](https://github.com/WorksApplications/Sudachi/blob/develop/docs/user_dict.md)).
+3. **Expect small tokenization differences.** sudachi.rs 0.7 aligned its lattice search, character-category handling and katakana OOV joining with Java Sudachi, so some inputs segment differently even with the same dictionary. Custom `char.def` files must replace `NOOOVBOW2` with `NOOOVBOW NOOOVEOW`.
+
+Other visible changes: `Dictionary#lookup` normalizes the query first (`ｶﾎﾞｽ` finds カボス), joined numerals and katakana carry an OOV `word_id` instead of `4294967295`, and a custom config that names no `char.def`/`unk.def` falls back to the files built into sudachi.rs.
+
 ## Dictionary management
 
 Rake tasks for managing Sudachi dictionaries:
 
 ```sh
-rake kabosu:install[small]      # Install a dictionary (VERSION=YYYYMMDD for a specific version)
+rake kabosu:install:small       # Install a dictionary (VERSION=YYYYMMDD for a specific version)
 rake kabosu:install_if_missing  # Same, but a no-op when a dictionary is already installed
 rake kabosu:list                # List installed dictionaries
 rake kabosu:versions            # Show available versions from GitHub
 rake kabosu:path                # Show path to best available dictionary
-rake kabosu:remove[small]       # Remove a dictionary (VERSION=YYYYMMDD for a specific version)
+rake kabosu:remove:small        # Remove a dictionary (VERSION=YYYYMMDD for a specific version)
 ```
 
 Dictionaries are stored in `~/.kabosu/dict/` by default. Set `KABOSU_DICT_DIR` to customize — useful for pointing at a Docker volume so the dictionary persists across deployments.
@@ -134,7 +148,7 @@ list = tokenizer.tokenize("国会議事堂前駅")
 list.surfaces
 list.first.part_of_speech
 
-# Dictionary prefix lookup
+# Dictionary prefix lookup (the query is normalized first, like tokenization)
 dict.lookup("東京都").surfaces
 
 # Morpheme split
@@ -146,6 +160,31 @@ Kabosu.split_sentences("東京都に住んでいる。大阪も好きだ。", ra
 Kabosu.split_sentences("長い文...", limit: 12, with_checker: true)
 ```
 
+### Dictionary entries and forms (sudachi.rs 0.7)
+
+```ruby
+m = tokenizer.tokenize("住んでいた").first   # 住ん
+m.dictionary_form_morpheme                   # the 住む entry itself: POS, reading, ids…
+tokenizer.tokenize("附属").first.normalized_form_morpheme.surface # => "付属"
+# A word that is its own form (and any OOV word) returns itself.
+
+# Every entry in the loaded dictionaries, lazily (millions for `full`)
+dict.entries.lazy.select(&:user?).first(10)
+
+# Exact matches by scanning every entry, including ones not indexed for
+# lookup. Slow on large dictionaries; prefer #lookup.
+dict.lookup_all_entries("ｶﾎﾞｽ").surfaces     # => ["カボス"]
+
+# Hand-made out-of-vocabulary morpheme; forms default to the surface
+dict.oov_morpheme(m.part_of_speech_id, "ズィーガン", reading: "ジーガン")
+
+# What tokenization actually sees after the input-text plugins
+dict.normalize("ｶﾎﾞｽ ＡＢＣ コンピュータ〜〜") # => "カボス abc コンピューター"
+
+# Compile a user dictionary against the installed system dictionary
+Kabosu.build_user_dict("names.csv", output: "names.dic")
+```
+
 ## Benchmarks
 
 Kabosu ships with a benchmark suite that measures tokenization throughput and compares the Ruby bindings against raw [sudachi.rs](https://github.com/WorksApplications/sudachi.rs).

data/lib/kabosu/dict_manager.rb

@@ -10,9 +10,16 @@ module Kabosu
     EDITION_PRIORITY = %w[full core small].freeze
     GITHUB_REPO = "WorksApplications/SudachiDict".freeze
     GITHUB_API = "https://api.github.com".freeze
+    # sudachi.rs 0.7 reads only V1 dictionaries, which open with this magic.
+    # Older (V0) files stay listed so they can be removed, but are never picked.
+    V1_MAGIC = "SudachiBinaryDic".b.freeze
+    # The first SudachiDict GitHub release carrying V1 assets. v20260723.1 is
+    # V1 too, but was published to PyPI only and has no assets to download.
+    MIN_VERSION = "20261015".freeze
 
     class DictNotFound < StandardError; end
     class DownloadError < StandardError; end
+    class UnsupportedVersion < StandardError; end
 
     # Default storage directory. Honors KABOSU_DICT_DIR so consumers can point
     # the gem at a Docker volume / shared mount without subclassing or threading
@@ -37,13 +44,19 @@ module Kabosu
     def install(edition = "core", version: nil)
       edition = validate_edition(edition)
       version ||= latest_version
+      validate_version(version)
 
       dest_dir = File.join(@dir, "sudachi-dictionary-#{version}")
       dic_path = File.join(dest_dir, "system_#{edition}.dic")
 
       if File.exist?(dic_path)
-        warn "Already installed: #{dic_path}"
-        return dic_path
+        if v1?(dic_path)
+          warn "Already installed: #{dic_path}"
+          return dic_path
+        end
+
+        warn "Replacing pre-V1 dictionary: #{dic_path}"
+        FileUtils.rm_f(dic_path)
       end
 
       FileUtils.mkdir_p(@dir)
@@ -73,6 +86,11 @@ module Kabosu
 
       raise DownloadError, "Expected #{dic_path} after extraction, but file not found" unless File.exist?(dic_path)
 
+      unless v1?(dic_path)
+        FileUtils.rm_f(dic_path)
+        raise UnsupportedVersion, "#{version}/#{edition} is not a V1 dictionary, which sudachi.rs 0.7+ requires"
+      end
+
       warn "Installed: #{dic_path}"
       dic_path
     end
@@ -87,7 +105,7 @@ module Kabosu
     #
     def install_if_missing(edition = "core", version: nil)
       edition = validate_edition(edition)
-      matching = installed.find do |d|
+      matching = readable.find do |d|
         d[:edition] == edition && (version.nil? || d[:version] == version)
       end
       return matching[:path] if matching
@@ -121,8 +139,13 @@ module Kabosu
     # Find the best available dictionary path.
     # Prefers: latest version, then full > core > small.
     def find(edition: nil)
-      candidates = installed
-      raise DictNotFound, "No dictionaries installed. Run: rake kabosu:install" if candidates.empty?
+      candidates = readable
+      if candidates.empty?
+        raise DictNotFound, "No dictionaries installed. Run: rake kabosu:install" if installed.empty?
+
+        raise DictNotFound, "Only pre-V1 dictionaries are installed, which sudachi.rs 0.7+ cannot read. " \
+                            "Run: rake kabosu:install (and rake kabosu:remove VERSION=... for the old ones)"
+      end
 
       if edition
         edition = validate_edition(edition)
@@ -144,6 +167,14 @@ module Kabosu
       best.is_a?(Hash) ? best[:path] : latest_version_dicts.first[:path]
     end
 
+    def readable
+      installed.select { |d| v1?(d[:path]) }
+    end
+
+    def v1?(path)
+      File.binread(path, V1_MAGIC.bytesize) == V1_MAGIC
+    end
+
     # ── Remove ──
 
     # Remove a specific dictionary edition, or an entire version.
@@ -189,6 +220,15 @@ module Kabosu
 
     private
 
+    def validate_version(version)
+      return if Gem::Version.new(version) >= Gem::Version.new(MIN_VERSION)
+
+      raise UnsupportedVersion, "SudachiDict #{version} predates the V1 format sudachi.rs 0.7+ reads; " \
+                                "use #{MIN_VERSION} or later"
+    rescue ArgumentError
+      raise UnsupportedVersion, "Malformed SudachiDict version: #{version.inspect}"
+    end
+
     def validate_edition(edition)
       edition = edition.to_s.downcase
       unless EDITIONS.include?(edition)

data/lib/kabosu/resources/char.def

@@ -174,6 +174,6 @@ CYRILLIC        1 1 0
 0x1F3FB..0x1F3FE ALL NOOOVBOW # emoji skin tone modifiers https://codepoints.net/U+1F3FF
 
 # Combination marks
-0x200C..0x200D ALL NOOOVBOW2 # Zero Width Non-Joiner/Joiner
+0x200C..0x200D ALL NOOOVBOW NOOOVEOW # Zero Width Non-Joiner/Joiner
 
 # END OF TABLE

data/lib/kabosu/tasks.rake

@@ -56,6 +56,19 @@ namespace :kabosu do
     end
   end
 
+  # ── User dictionaries ──
+
+  desc "Build a user dictionary: CSV=a.csv[,b.csv] OUTPUT=user.dic [SYSTEM_DICT=path] " \
+       "(defaults to the best installed dictionary)"
+  task :build_user_dict do
+    require "kabosu"
+    csv = ENV.fetch("CSV") { abort "CSV=path/to/lexicon.csv is required" }.split(",")
+    output = ENV.fetch("OUTPUT") { abort "OUTPUT=path/to/user.dic is required" }
+    system_dict = ENV.fetch("SYSTEM_DICT") { Kabosu::Dictionary.path }
+    Kabosu.build_user_dict(csv, output: output, system_dict: system_dict)
+    puts "Built #{output} against #{system_dict}"
+  end
+
   # ── Info ──
 
   desc "List installed Sudachi dictionaries"
@@ -70,7 +83,8 @@ namespace :kabosu do
       puts
       dicts.each do |d|
         size_mb = (File.size(d[:path]).to_f / 1024 / 1024).round(1)
-        puts "  #{d[:version]} / #{d[:edition].ljust(5)} (#{size_mb} MB)"
+        unreadable = manager.v1?(d[:path]) ? "" : "  [pre-V1, unreadable by this kabosu]"
+        puts "  #{d[:version]} / #{d[:edition].ljust(5)} (#{size_mb} MB)#{unreadable}"
         puts "    #{d[:path]}"
       end
     end

data/lib/kabosu/version.rb

@@ -1,3 +1,3 @@
 module Kabosu
-  VERSION = "0.6.11.4".freeze
+  VERSION = "0.7.0.1".freeze
 end

data/lib/kabosu.rb

@@ -105,6 +105,14 @@ module Kabosu
 
       def map_dictionary_init_error(error, config:, system_dict:)
         message = error.message
+        if message.include?("V0 version")
+          return DictionaryError.new(
+            "#{message}: a system or user dictionary is in the pre-V1 format, which sudachi.rs 0.7+ cannot read. " \
+            "Install SudachiDict #{DictManager::MIN_VERSION} or later (rake kabosu:install) " \
+            "and rebuild user dictionaries against it."
+          )
+        end
+
         if (config && system_dict.nil?) || message.match?(/config|setting\.json|json/i)
           ConfigError.new(message)
         else
@@ -148,6 +156,41 @@ module Kabosu
     rescue RuntimeError => e
       raise LookupError.new(e.message), cause: e
     end
+
+    # Every public dictionary entry as a standalone Morpheme (offsets cover its
+    # own headword). Lazy: a `full` dictionary holds millions of entries, so
+    # chain with `.lazy`, `.first(n)` or `.each` rather than `.to_a`.
+    #
+    #   dict.entries.lazy.select { _1.user? }.first(10)
+    def entries(&block)
+      return enum_for(:entries) unless block
+
+      _each_entry(&block)
+      self
+    end
+
+    # Like #lookup, but exact matches only and found by scanning every entry,
+    # so it also finds entries that are not indexed for lookup. Slow.
+    def lookup_all_entries(surface)
+      raise ArgumentError, "surface must be a String" unless surface.is_a?(String)
+
+      MorphemeList.new(_lookup_all_entries(surface))
+    rescue RuntimeError => e
+      raise LookupError.new(e.message), cause: e
+    end
+
+    # Builds an out-of-vocabulary Morpheme by hand. The forms default to the
+    # surface. pos_id comes from Morpheme#part_of_speech_id.
+    #
+    #   dict.oov_morpheme(pos_id, "ズィーガンボルト", reading: "ズィーガンボルト")
+    def oov_morpheme(pos_id, surface, reading: nil, normalized_form: nil, dictionary_form: nil)
+      raise ArgumentError, "pos_id must be an Integer" unless pos_id.is_a?(Integer)
+      raise ArgumentError, "surface must be a String" unless surface.is_a?(String)
+
+      _oov_morpheme(pos_id, surface, reading, normalized_form, dictionary_form)
+    rescue RangeError, RuntimeError => e
+      raise ArgumentError, "invalid part-of-speech id #{pos_id}: #{e.message}"
+    end
   end
 
   # ── Tokenizer: wrap output in MorphemeList ──
@@ -179,6 +222,23 @@ module Kabosu
     end
   end
 
+  # Compiles user-dictionary CSV(s) into a binary user dictionary. The result
+  # loads only next to the system dictionary it was built against, so rebuild
+  # whenever that changes (including the move to sudachi.rs 0.7).
+  #
+  #   Kabosu.build_user_dict("names.csv", output: "names.dic")
+  #   Kabosu::Dictionary.new(system_dict: Kabosu::Dictionary.path, user_dicts: ["names.dic"])
+  def self.build_user_dict(inputs, output:, system_dict: Dictionary.path, description: nil)
+    inputs = Array(inputs)
+    raise ArgumentError, "inputs must be String paths" unless inputs.all?(String)
+    raise ArgumentError, "output must be a String" unless output.is_a?(String)
+
+    _build_user_dict(inputs, system_dict, output, description)
+    output
+  rescue RuntimeError => e
+    raise DictionaryError.new(e.message), cause: e
+  end
+
   def self.split_sentences(text, limit: nil, with_checker: false, ranges: false, dictionary: nil)
     raise ArgumentError, "text must be a String" unless text.is_a?(String)
     raise ArgumentError, "limit must be an Integer or nil" unless limit.nil? || limit.is_a?(Integer)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kabosu
 version: !ruby/object:Gem::Version
-  version: 0.6.11.4
+  version: 0.7.0.1
 platform: aarch64-linux
 authors:
 - davafons
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2026-09-13 00:00:00.000000000 Z
+date: 2026-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rubyzip