lindera 3.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a8fae246f910033fc6c1b40d3ba6d7cf36df5911b7bd8d84eac97e77f3a9eb29
+  data.tar.gz: 248e5eeb5a3824e05ebabd104579c558a36a53a7c0b82b4b314b68b9f971459f
+SHA512:
+  metadata.gz: 00a9bf21d2b3a68119ff442e64e42b0de5f1579be4c631450c5715ac1a3e1b5d68ae46757e03fd6140661e49633d2f76bea34fbb7965a3da9fc54b1b440a659a
+  data.tar.gz: 6e2b6887e847433f64751770f47eca868aa3cabcf0c91666618169399abf4746962369ae6d22acdb508a8a0ff62a809eaed2a63722258f77874fb139d52ba51d

data/Cargo.toml

@@ -0,0 +1,40 @@
+[package]
+name = "lindera-ruby"
+version = { workspace = true }
+edition = { workspace = true }
+description = "A Ruby binding for Lindera."
+documentation = "https://docs.rs/lindera-ruby"
+homepage = { workspace = true }
+repository = { workspace = true }
+readme = "README.md"
+keywords = ["morphological", "analysis", "library", "ruby", "binding"]
+categories = { workspace = true }
+license = { workspace = true }
+
+[lib]
+name = "lindera_ruby"
+crate-type = ["cdylib", "lib"]
+
+[features]
+embed-ipadic = ["lindera/embed-ipadic"] # Include Japanese dictionary (IPADIC)
+embed-ipadic-neologd = [
+    "lindera/embed-ipadic-neologd",
+] # Include Japanese dictionary (IPADIC NEologd)
+embed-unidic = ["lindera/embed-unidic"] # Include Japanese dictionary (UniDic)
+embed-ko-dic = ["lindera/embed-ko-dic"] # Include Korean dictionary (ko-dic)
+embed-cc-cedict = [
+    "lindera/embed-cc-cedict",
+] # Include Chinese dictionary (CC-CEDICT)
+embed-jieba = ["lindera/embed-jieba"] # Include Chinese dictionary (Jieba)
+embed-cjk = [
+    "lindera/embed-cjk",
+] # Embed CJK dictionaries (IPADIC, ko-dic, Jieba)
+train = ["lindera/train"] # Enable training functionality
+default = ["train"] # No directories included
+
+[dependencies]
+lindera = { workspace = true }
+magnus = { version = "0.8.2", features = ["rb-sys"] }
+num_cpus = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
+serde_json = { workspace = true }

data/README.md

@@ -0,0 +1,78 @@
+# lindera-ruby
+
+Ruby binding for [Lindera](https://github.com/lindera/lindera), a morphological analysis engine for CJK text.
+
+## Overview
+
+lindera-ruby provides a Ruby interface to the Lindera morphological analysis engine, supporting Japanese, Korean, and Chinese text analysis.
+
+- **Multi-language Support**: Japanese (IPADIC, IPADIC-NEologd, UniDic), Korean (ko-dic), Chinese (CC-CEDICT, Jieba)
+- **Character Filters**: Text preprocessing with mapping, regex, Unicode normalization, and Japanese iteration mark handling
+- **Token Filters**: Post-processing filters including lowercase, length filtering, stop words, and Japanese-specific filters
+- **Flexible Configuration**: Configurable tokenization modes and penalty settings
+- **Metadata Support**: Complete dictionary schema and metadata management
+- **Training & Export** (optional): Train custom morphological analysis models from corpus data
+
+## Requirements
+
+- Ruby >= 3.1
+- Rust >= 1.85
+
+## Dictionary
+
+Pre-built dictionaries are available from [GitHub Releases](https://github.com/lindera/lindera/releases).
+Download a dictionary archive (e.g. `lindera-ipadic-*.zip`) and extract it to a local path.
+
+## Install
+
+```bash
+cd lindera-ruby
+bundle install
+bundle exec rake compile
+```
+
+## Usage
+
+```ruby
+require "lindera"
+
+# Load dictionary from a local path (download from GitHub Releases)
+dictionary = Lindera.load_dictionary("/path/to/ipadic")
+
+# Create a tokenizer
+tokenizer = Lindera::Tokenizer.new(dictionary, "normal", nil)
+
+# Tokenize text
+tokens = tokenizer.tokenize("関西国際空港")
+tokens.each do |token|
+  puts "#{token.surface}: #{token.details&.join(', ')}"
+end
+```
+
+### Using TokenizerBuilder
+
+```ruby
+require "lindera"
+
+builder = Lindera::TokenizerBuilder.new
+builder.set_mode("normal")
+builder.set_dictionary("/path/to/ipadic")
+
+# Add filters
+builder.append_character_filter("unicode_normalize", { "kind" => "nfkc" })
+builder.append_token_filter("lowercase", nil)
+
+tokenizer = builder.build
+tokens = tokenizer.tokenize("テスト")
+```
+
+## Test
+
+```bash
+bundle exec rake compile
+bundle exec rake test
+```
+
+## License
+
+MIT

data/ext/lindera_ruby/extconf.rb

@@ -0,0 +1,8 @@
+require "rb_sys/mkmf"
+
+create_rust_makefile("lindera/lindera_ruby") do |r|
+  # Pass embed features via LINDERA_FEATURES environment variable
+  # e.g., LINDERA_FEATURES="embed-ipadic" bundle exec rake compile
+  features = ENV.fetch("LINDERA_FEATURES", "").split(",").map(&:strip).reject(&:empty?)
+  r.features = features unless features.empty?
+end

data/lib/lindera.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+begin
+  require 'lindera/lindera_ruby'
+rescue LoadError
+  require 'lindera_ruby'
+end

metadata

@@ -0,0 +1,100 @@
+--- !ruby/object:Gem::Specification
+name: lindera
+version: !ruby/object:Gem::Version
+  version: 3.0.0
+platform: ruby
+authors:
+- Lindera contributors
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rb_sys
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+description: Ruby bindings for Lindera, a morphological analysis library for CJK text
+  (Japanese, Korean, Chinese).
+executables: []
+extensions:
+- ext/lindera_ruby/extconf.rb
+extra_rdoc_files: []
+files:
+- Cargo.toml
+- README.md
+- ext/lindera_ruby/extconf.rb
+- lib/lindera.rb
+homepage: https://github.com/lindera/lindera
+licenses:
+- MIT
+metadata: {}
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.6
+specification_version: 4
+summary: Ruby bindings for Lindera morphological analysis engine
+test_files: []