kabosu 0.1.1 → 0.6.10.dev.20260225.4c46cc6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7162c76ded65b48d00129efb28bea2eb01bd7a2381003d2e95f81ac341365870
-  data.tar.gz: 40e0177fa9b314926814d6eff293c2a7afa71e8ebe0ab52edb50937257587b7c
+  metadata.gz: 777e090671b61cde329346c6179b4fb89373d52f93f89dff9cacbf3fe31806b7
+  data.tar.gz: 17f5680cbcf45ad4cbcdac0c7c8e28827fd86ec480d999827eb2da1a192e75fd
 SHA512:
-  metadata.gz: bc3805c482f2434563d71eae12bb4a6a94b8793d2320d3d400ed18bb86c93eecdfd95783ceb76fbfa501c7ad2b5d26c472aca82846c455136a4ff150408092d2
-  data.tar.gz: 21dd0f7358f4808ec0b9c1dc6493d8e3bd867c73b8d57c650eda88cca0d3d42715c9a99a8004cb4be230d752822bf85af10a854fc3c7843a3abeba2756150b86
+  metadata.gz: ca0311c2bc431ffb2cfb4a144933db1f59530624aa29f87106ed8f97ed87f5bc858debe4168ab6a5af4b693627a2eb72b25060aaa2be5778f367b7830b8a9614
+  data.tar.gz: e31111ad3f7ce865d667702b977db999dc4519c8f894ef879ab9191ccf6bc62dc6c65d6233f79e380f76b697c9f23451ec248a01bbc8568e38fdd7c5b79956ae

data/LICENSE

@@ -0,0 +1,191 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to the Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by the Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding any notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   Copyright 2025 davafons
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -1,49 +1,98 @@
-# Kabosu
+<p align="center">
+  <img src="logo.png" alt="Kabosu" width="150">
+</p>
+
+<h1 align="center">Kabosu</h1>
+
+<p align="center">
+  <a href="https://rubygems.org/gems/kabosu"><img src="https://img.shields.io/gem/v/kabosu" alt="Gem Version"></a>
+  <a href="https://github.com/davafons/kabosu/actions/workflows/edge.yml"><img src="https://github.com/davafons/kabosu/actions/workflows/edge.yml/badge.svg" alt="CI"></a>
+  <a href="https://github.com/davafons/kabosu/blob/main/LICENSE"><img src="https://img.shields.io/github/license/davafons/kabosu" alt="License"></a>
+  <a href="https://rubygems.org/gems/kabosu"><img src="https://img.shields.io/gem/dt/kabosu" alt="Downloads"></a>
+</p>
 
 Ruby bindings for [sudachi.rs](https://github.com/WorksApplications/sudachi.rs), a Rust implementation of the [Sudachi](https://github.com/WorksApplications/Sudachi) Japanese morphological analyzer.
 
-## Requirements
+## Usage
 
-- Ruby >= 3.1
-- Rust toolchain (for compiling the native extension)
+```ruby
+require "kabosu"
+
+# Explicit dictionary + tokenizer lifecycle
+dict = Kabosu::Dictionary.new(system_dict: Kabosu::Dictionary.path)
+tokenizer = dict.create(mode: :c)
+
+# Tokenize Japanese text
+morphemes = Kabosu.tokenize("東京都に住んでいる", tokenizer: tokenizer)
+
+# Bulk accessors for quick extraction
+morphemes.surfaces          # => ["東京都", "に", "住ん", "で", "いる"]
+morphemes.readings          # => ["トウキョウト", "ニ", "スン", "デ", "イル"]
+morphemes.dictionary_forms  # => ["東京都", "に", "住む", "で", "居る"]
+
+# Each morpheme exposes rich linguistic detail
+morpheme = morphemes.first
+morpheme.surface             # => "東京都"          - surface form (as it appears in text)
+morpheme.part_of_speech      # => ["名詞", "固有名詞", "地名", "一般"] — part-of-speech tags
+morpheme.part_of_speech_id   # => 5                - numeric POS id
+morpheme.dictionary_form     # => "東京都"          - base/dictionary form
+morpheme.normalized_form     # => "東京都"          - normalized form
+morpheme.reading_form        # => "トウキョウト"     - phonetic reading
+morpheme.oov?                # => false            - out-of-vocabulary?
+morpheme.dictionary_id       # => 0                - source dictionary id
+morpheme.word_id             # => 544373           - internal word id
+morpheme.synonym_group_ids   # => []               - synonym group ids
+morpheme.dictionary_form_word_id # => -1           - dictionary-form word id
+morpheme.head_word_length    # => 3                - head word length in codepoints
+morpheme.a_unit_split        # => [123, 456]       - split-A word ids
+morpheme.b_unit_split        # => []               - split-B word ids
+morpheme.word_structure      # => [123, 456]       - word-structure ids
+morpheme.total_cost          # => 5765             - morphological analysis cost
+morpheme.begin               # => 0                - start byte offset
+morpheme.end                 # => 9                - end byte offset
+morpheme.begin_c             # => 0                - start character offset
+morpheme.end_c               # => 3                - end character offset
+morpheme.system?             # => true             - from system dictionary?
+morpheme.user?               # => false            - from user dictionary?
+```
 
 ## Installation
 
+- Ruby >= 3.1
+- Rust toolchain (for compiling the native extension)
+
 Add to your Gemfile:
 
 ```ruby
 gem "kabosu"
 ```
 
-Then install and download a Sudachi dictionary:
+Then install and download a [Sudachi dictionary](https://github.com/WorksApplications/SudachiDict):
 
 ```sh
 bundle install
 bundle exec rake kabosu:install[small]  # or core, full
 ```
 
-Dictionary editions (from smallest to largest): `small`, `core`, `full`.
-
-## Usage
+Dictionary editions (from smallest to largest): `small`, `core`, `full`. See the [SudachiDict documentation](https://github.com/WorksApplications/SudachiDict?tab=readme-ov-file#dictionary-types) for details on the differences between editions.
 
-```ruby
-require "kabosu"
-
-# Tokenize Japanese text (auto-discovers installed dictionary)
-morphemes = Kabosu.tokenize("東京都に住んでいる")
+## Dictionary management
 
-morphemes.surfaces       # => ["東京都", "に", "住ん", "で", "いる"]
-morphemes.readings       # => ["トウキョウト", "ニ", "スン", "デ", "イル"]
-morphemes.dictionary_forms # => ["東京都", "に", "住む", "で", "居る"]
+Rake tasks for managing Sudachi dictionaries:
 
-morphemes.each do |m|
-  puts "#{m.surface}\t#{m.part_of_speech.join(',')}\t#{m.reading_form}"
-end
+```sh
+rake kabosu:install[small]     # Install a dictionary (VERSION=YYYYMMDD for a specific version)
+rake kabosu:list               # List installed dictionaries
+rake kabosu:versions           # Show available versions from GitHub
+rake kabosu:path               # Show path to best available dictionary
+rake kabosu:remove[small]      # Remove a dictionary (VERSION=YYYYMMDD for a specific version)
 ```
 
-### Tokenization modes
+Dictionaries are stored in `~/.kabosu/dict/` by default. Set `KABOSU_DICT_DIR` to customize.
+
+## Tokenization modes
 
-Sudachi provides three split modes:
+Sudachi provides three [split modes](https://github.com/WorksApplications/Sudachi?tab=readme-ov-file#the-modes-of-splitting):
 
 | Mode | Description |
 |------|-------------|
@@ -52,43 +101,121 @@ Sudachi provides three split modes:
 | `C` | Named entity units (default) |
 
 ```ruby
-Kabosu.tokenize("東京都", mode: "A").surfaces  # => ["東京", "都"]
-Kabosu.tokenize("東京都", mode: "C").surfaces  # => ["東京都"]
+dict = Kabosu::Dictionary.new(system_dict: Kabosu::Dictionary.path)
+tok_a = dict.create(mode: :a)
+tok_c = dict.create(mode: :c)
+tok_a.tokenize("東京都").surfaces  # => ["東京", "都"]
+tok_c.tokenize("東京都").surfaces  # => ["東京都"]
 ```
 
-### Direct API
+Modes are symbols only (`:a`, `:b`, `:c` or `Kabosu::MODE_A/B/C`).
+Invalid modes now raise `ArgumentError` (for example, `"A"`).
 
-For more control, create a dictionary and tokenizer directly:
+## Dictionary and Tokenizer Internal API
+
+For more control over dictionary and tokenizer configuration, create them directly:
 
 ```ruby
-dict = Kabosu::Dictionary.new
-tokenizer = dict.create("C")
+dict = Kabosu::Dictionary.new(
+  system_dict: "/path/to/custom/system.dic",
+  user_dicts: ["/path/to/domain.dic", "/path/to/names.dic"]
+)
+tokenizer = dict.create(mode: :c, fields: %i[surface pos_id reading_form])
+
 morphemes = tokenizer.tokenize("国会議事堂前駅")
+# MorphemeList is lazy: morphemes are hydrated on first indexed/iterated access.
+# surfaces uses a fast path and does not force full morpheme hydration.
+morphemes.surfaces
+morphemes.first.part_of_speech
+
+# Lexicon lookup (prefix matches from position 0), returns MorphemeList
+dict.lookup("東京都").surfaces
+
+# Morpheme split returns MorphemeList
+m = tokenizer.tokenize("東京都").first
+m.split(mode: :a).surfaces
+
+# Native bulk extraction helpers (fewer Ruby<->Rust crossings)
+tokenizer.tokenize_surfaces("東京都に住んでいる")
+tokenizer.tokenize_readings("東京都に住んでいる")
+tokenizer.tokenize_dictionary_forms("東京都に住んでいる")
+tokenizer.tokenize_normalized_forms("東京都に住んでいる")
+
+# Sentence splitting options
+Kabosu.split_sentences("東京都に住んでいる。大阪も好きだ。", ranges: true)
+Kabosu.split_sentences("長い文...", limit: 12, with_checker: true)
+
+# ranges: true returns SentenceRange objects
+ranges = Kabosu.split_sentences("東京都に住んでいる。", ranges: true)
+ranges.first.start
+ranges.first.end
+ranges.first.text
 ```
 
-## Dictionary management
+Dictionary initialization failures raise typed errors:
+- `Kabosu::ConfigError` for configuration issues
+- `Kabosu::DictionaryError` for dictionary loading issues
 
-Rake tasks for managing Sudachi dictionaries:
+Runtime failures in analysis APIs are also typed:
+- `Kabosu::TokenizationError` for tokenization/split failures
+- `Kabosu::SentenceSplitError` for sentence splitter failures
+- `Kabosu::LookupError` for dictionary lookup failures
+
+## Public API Contract
+
+| API | Parameters | Return | Notes |
+|---|---|---|---|
+| `Kabosu::Dictionary.new` | `config: String?`, `system_dict: String?`, `user_dicts: Array<String>?` | `Kabosu::Dictionary` | One of `config` or `system_dict` is required |
+| `Dictionary#create` | `mode: :a|:b|:c`, `fields: Array<String\|Symbol>?`, `debug: bool`, `projection: nil` | `Kabosu::Tokenizer` | Unknown kwargs raise `ArgumentError`; `projection` currently raises `NotImplementedError` |
+| `Dictionary#lookup` | `text: String` | `Kabosu::MorphemeList` | Prefix lookup from byte offset 0 |
+| `Tokenizer#tokenize` | `text: String` | `Kabosu::MorphemeList` | Lazy morpheme hydration; raises `Kabosu::TokenizationError` on native failures |
+| `Tokenizer#tokenize_surfaces/readings/dictionary_forms/normalized_forms` | `text: String` | `Array<String>` | Raises `Kabosu::TokenizationError` on native failures |
+| `Morpheme#split` | `mode: :a|:b|:c`, `add_single: bool` | `Kabosu::MorphemeList` | Standardized with `tokenize` return type |
+| `Kabosu.split_sentences` | `text: String`, `limit: Integer?`, `with_checker: bool`, `ranges: bool`, `dictionary: String?` | `Array<String>` or `Array<Kabosu::SentenceRange>` | `limit` must be `>= 1` |
+| `Kabosu.tokenize` | `text: String`, `tokenizer: Kabosu::Tokenizer` | `Kabosu::MorphemeList` | No hidden global tokenizer cache |
+
+## Benchmarks
+
+Kabosu ships with a benchmark suite that measures tokenization throughput and compares the Ruby bindings against raw [sudachi.rs](https://github.com/WorksApplications/sudachi.rs).
+
+This benchmark uses [Wagahai wa Neko de Aru](https://www.aozora.gr.jp/cards/000148/card789.html) (I Am a Cat) by Natsume Soseki, sourced from [Aozora Bunko](https://www.aozora.gr.jp/) (public domain) as the source text. ~958 KB of Japanese prose, 2,256 lines as input.
+
+### Results
+
+Measured on an AMD Ryzen 7 5800X, `full` dictionary edition, Ruby 3.4, Rust 1.84:
+
+| Scenario | Rust | Ruby | Ratio |
+|---|---|---|---|
+| split_sentences | 1.597s | 1.677s | 1.0x |
+| tokenize (mode C) | 3.274s | 4.034s | 1.2x |
+| tokenize (mode A) | 3.429s | 4.273s | 1.2x |
+| tokenize (mode B) | 3.465s | 4.297s | 1.2x |
+| **Throughput** | **2.66 MB/s** | **2.18 MB/s** | **1.2x** |
+
+The Ruby bindings add ~20% overhead over raw Rust, primarily from FFI boundary crossings and Ruby object allocation for each morpheme.
+
+To reproduce these results, run:
 
 ```sh
-rake kabosu:install[small]     # Install a dictionary (VERSION=YYYYMMDD for a specific version)
-rake kabosu:list               # List installed dictionaries
-rake kabosu:versions           # Show available versions from GitHub
-rake kabosu:path               # Show path to best available dictionary
-rake kabosu:remove[small]      # Remove a dictionary (VERSION=YYYYMMDD for a specific version)
+bundle exec ruby bench/start
 ```
 
-Dictionaries are stored in `~/.kabosu/dict/` by default. Set `KABOSU_DICT_DIR` to customize.
+To generate flamegraph SVGs alongside the benchmark:
 
-## Development
+```sh
+bundle exec ruby bench/start --profile
+```
+
+This records both the Rust and Ruby runs with `perf` and produces interactive SVGs (`bench/flamegraph-rust.svg`, `bench/flamegraph-ruby.svg`). Open them in a browser to explore.
+
+## Contributing
 
 ```sh
 bundle install
-bundle exec rake compile   # Build the native extension
-bundle exec rake test      # Run tests
-bundle exec rake           # Compile + test
-```
 
-## License
+bundle exec rake kabosu:install # Install Sudachi dictionary
 
-MIT
+bundle exec rake compile        # Build the native extension  
+bundle exec rake test           # Run tests
+bench/start                     # Run benchmarks
+```

data/ext/kabosu/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kabosu"
-version = "0.1.1"
+version = "0.6.10"
 edition = "2021"
 publish = false
 
@@ -9,4 +9,4 @@ crate-type = ["cdylib"]
 
 [dependencies]
 magnus = { version = "0.8", features = ["rb-sys"] }
-sudachi = { git = "https://github.com/WorksApplications/sudachi.rs", branch = "develop" }
+sudachi = { git = "https://github.com/WorksApplications/sudachi.rs", tag = "v0.6.10" }

data/ext/kabosu/src/dictionary.rs

@@ -0,0 +1,104 @@
+use magnus::{Error, RArray, Ruby, Value};
+use std::path::Path;
+use std::sync::Arc;
+use sudachi::config::{Config, ConfigBuilder};
+use sudachi::dic::dictionary::JapaneseDictionary;
+
+use crate::errors::sudachi_error;
+use crate::morpheme::{rb_morpheme_from_data, MorphemeData};
+use crate::parsing::{parse_mode, parse_subset, parse_user_dicts};
+use crate::tokenizer::RbTokenizer;
+
+fn build_dictionary_config(
+    config_path: Option<String>,
+    system_dict_path: Option<String>,
+    user_dicts: Vec<String>,
+) -> Result<Config, Error> {
+    let mut builder = ConfigBuilder::from_opt_file(config_path.as_deref().map(Path::new))
+        .map_err(sudachi_error)?;
+
+    if let Some(system_dict_path) = system_dict_path {
+        builder = builder.system_dict(system_dict_path);
+    }
+
+    for user_dict in user_dicts {
+        builder = builder.user_dict(user_dict);
+    }
+
+    Ok(builder.build())
+}
+
+#[magnus::wrap(class = "Kabosu::Dictionary")]
+pub(crate) struct RbDictionary {
+    inner: Arc<JapaneseDictionary>,
+}
+
+impl RbDictionary {
+    pub(crate) fn new(
+        config_path: Option<String>,
+        system_dict_path: Option<String>,
+        user_dicts_value: Option<Value>,
+    ) -> Result<Self, Error> {
+        let user_dicts = parse_user_dicts(user_dicts_value)?;
+        let cfg = build_dictionary_config(config_path, system_dict_path, user_dicts)?;
+        let dict = JapaneseDictionary::from_cfg(&cfg).map_err(sudachi_error)?;
+
+        Ok(Self {
+            inner: Arc::new(dict),
+        })
+    }
+
+    pub(crate) fn lookup(&self, text: String) -> Result<RArray, Error> {
+        let ruby = Ruby::get().unwrap();
+        let ary = ruby.ary_new();
+        let lexicon = self.inner.lexicon();
+
+        for entry in lexicon.lookup(text.as_bytes(), 0) {
+            let info = lexicon.get_word_info(entry.word_id).map_err(sudachi_error)?;
+            let surface_slice = text.get(..entry.end).ok_or_else(|| {
+                sudachi_error(format!(
+                    "lookup returned invalid UTF-8 boundary: end={}",
+                    entry.end
+                ))
+            })?;
+
+            let data = MorphemeData {
+                surface: surface_slice.to_string(),
+                pos_id: info.pos_id(),
+                word_id_raw: entry.word_id.as_raw(),
+                is_oov: entry.word_id.is_oov(),
+                dictionary_id: if entry.word_id.is_oov() {
+                    -1
+                } else {
+                    entry.word_id.dic() as i32
+                },
+                is_system: entry.word_id.is_system(),
+                is_user: entry.word_id.is_user(),
+                begin: 0,
+                end: entry.end,
+                begin_c: 0,
+                end_c: surface_slice.chars().count(),
+                total_cost: 0,
+            };
+            ary.push(rb_morpheme_from_data(data, self.inner.clone(), false))?;
+        }
+
+        Ok(ary)
+    }
+
+    pub(crate) fn create(
+        &self,
+        mode_str: Option<String>,
+        fields: Option<Value>,
+        debug: Option<bool>,
+    ) -> Result<RbTokenizer, Error> {
+        let subset = parse_subset(fields)?;
+        let mode = parse_mode(mode_str.as_deref())?;
+        Ok(RbTokenizer::new(
+            self.inner.clone(),
+            mode,
+            subset,
+            debug.unwrap_or(false),
+        ))
+    }
+}

data/ext/kabosu/src/errors.rs

@@ -0,0 +1,12 @@
+use magnus::{Error, Ruby};
+
+pub(crate) fn arg_error(msg: impl Into<String>) -> Error {
+    Error::new(Ruby::get().unwrap().exception_arg_error(), msg.into())
+}
+
+pub(crate) fn sudachi_error(e: impl std::fmt::Display) -> Error {
+    Error::new(
+        Ruby::get().unwrap().exception_runtime_error(),
+        e.to_string(),
+    )
+}